def test():
parser = argparse.ArgumentParser(description='Train VAE.')
parser.add_argument('-c', '--config', help='Config file.')
args = parser.parse_args()
print(args)
c = json.load(open(args.config))
print(c)
# clear param store
pyro.clear_param_store()
# batch_size = 64
# root_dir = r'D:\projects\trading\mlbootcamp\tickers'
# series_length = 60
lookback = 50 # 160
input_dim = 1
test_start_date = datetime.strptime(
c['test_start_date'], '%Y/%m/%d') if c['test_start_date'] else None
test_end_date = datetime.strptime(
c['test_end_date'], '%Y/%m/%d') if c['test_end_date'] else None
min_sequence_length_test = 2 * (c['series_length'] + lookback)
max_n_files = None
out_path = Path(c['out_dir'])
out_path.mkdir(exist_ok=True)
# load_path = 'out_saved/checkpoint_0035.pt'
dataset_test = create_ticker_dataset(
c['in_dir'],
c['series_length'],
lookback,
min_sequence_length_test,
start_date=test_start_date,
end_date=test_end_date,
fixed_start_date=True,
normalised_returns=c['normalised_returns'],
max_n_files=max_n_files)
test_loader = DataLoader(dataset_test,
batch_size=c['batch_size'],
shuffle=False,
num_workers=0,
drop_last=True)
# N_train_data = len(dataset_train)
N_test_data = len(dataset_test)
# N_mini_batches = N_train_data // c['batch_size']
# N_train_time_slices = c['batch_size'] * N_mini_batches
print(f'N_test_data: {N_test_data}')
# setup the VAE
vae = VAE(c['series_length'], z_dim=c['z_dim'], use_cuda=c['cuda'])
# setup the optimizer
# adam_args = {"lr": args.learning_rate}
# optimizer = Adam(adam_args)
# setup the inference algorithm
# elbo = JitTrace_ELBO() if args.jit else Trace_ELBO()
# svi = SVI(vae.model, vae.guide, optimizer, loss=elbo)
if c['checkpoint_load']:
checkpoint = torch.load(c['checkpoint_load'])
vae.load_state_dict(checkpoint['model_state_dict'])
# optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
if 1:
find_similar(vae, test_loader, c['cuda'])
# Visualise first batch.
batch = next(iter(test_loader))
x = batch['series']
if c['cuda']:
x = x.cuda()
x = x.float()
x_reconst = vae.reconstruct_img(x)
x = x.cpu().numpy()
x_reconst = x_reconst.cpu().detach().numpy()
n = min(5, x.shape[0])
fig, axes = plt.subplots(n, 1, squeeze=False)
for s in range(n):
ax = axes[s, 0]
ax.plot(x[s])
ax.plot(x_reconst[s])
fig.savefig(out_path / f'test_batch.png')
def main(args):
# setup logging
log = get_logger(args.log)
log(args)
root_dir = r'D:\projects\trading\mlbootcamp\tickers'
series_length = 60
lookback = 50 #160
input_dim = 1
train_start_date = datetime.date(2010, 1, 1)
train_end_date = datetime.date(2016, 1, 1)
test_start_date = train_end_date
test_end_date = datetime.date(2019, 1, 1)
min_sequence_length_train = 2 * (series_length + lookback)
min_sequence_length_test = 2 * (series_length + lookback)
dataset_train = create_ticker_dataset(root_dir,
series_length,
lookback,
min_sequence_length_train,
start_date=train_start_date,
end_date=train_end_date)
dataset_test = create_ticker_dataset(root_dir,
series_length,
lookback,
min_sequence_length_test,
start_date=test_start_date,
end_date=test_end_date)
dataloader_train = DataLoader(dataset_train,
batch_size=args.mini_batch_size,
shuffle=True,
num_workers=0,
drop_last=True)
dataloader_test = DataLoader(dataset_test,
batch_size=args.mini_batch_size,
shuffle=False,
num_workers=0,
drop_last=True)
N_train_data = len(dataset_train)
N_test_data = len(dataset_test)
N_mini_batches = N_train_data // args.mini_batch_size
N_train_time_slices = args.mini_batch_size * N_mini_batches
print(f'N_train_data: {N_train_data}, N_test_data: {N_test_data}')
# how often we do validation/test evaluation during training
val_test_frequency = 50
# the number of samples we use to do the evaluation
n_eval_samples = 1
# instantiate the dmm
dmm = DMM(input_dim=input_dim,
rnn_dropout_rate=args.rnn_dropout_rate,
num_iafs=args.num_iafs,
iaf_dim=args.iaf_dim,
use_cuda=args.cuda)
# setup optimizer
adam_params = {
"lr": args.learning_rate,
"betas": (args.beta1, args.beta2),
"clip_norm": args.clip_norm,
"lrd": args.lr_decay,
"weight_decay": args.weight_decay
}
adam = ClippedAdam(adam_params)
# setup inference algorithm
elbo = JitTrace_ELBO() if args.jit else Trace_ELBO()
svi = SVI(dmm.model, dmm.guide, adam, loss=elbo)
# if checkpoint files provided, load model and optimizer states from disk before we start training
if args.load_opt != '' and args.load_model != '':
load_checkpoint(dmm, adam, log)
#################
# TRAINING LOOP #
#################
times = [time.time()]
for epoch in range(args.num_epochs):
print(f'Starting epoch {epoch}.')
# accumulator for our estimate of the negative log likelihood (or rather -elbo) for this epoch
epoch_nll = 0.0
# prepare mini-batch subsampling indices for this epoch
shuffled_indices = torch.randperm(N_train_data)
# process each mini-batch; this is where we take gradient steps
dmm.train()
for which_mini_batch in range(N_mini_batches):
print(
f'Epoch {epoch} of {args.num_epochs}, Batch {which_mini_batch} of {N_mini_batches}.'
)
mini_batch = next(iter(dataloader_train))
epoch_nll += process_minibatch(svi, epoch, mini_batch,
N_mini_batches, which_mini_batch,
shuffled_indices)
# if specified, save model and optimizer states to disk every checkpoint_freq epochs
if 1: #args.checkpoint_freq > 0 and epoch > 0 and epoch % args.checkpoint_freq == 0:
save_checkpoint(dmm, adam, log)
# report training diagnostics
times.append(time.time())
epoch_time = times[-1] - times[-2]
log("[training epoch %04d] %.4f \t\t\t\t(dt = %.3f sec)" %
(epoch, epoch_nll / N_train_time_slices, epoch_time))
# do evaluation on test and validation data and report results
if val_test_frequency > 0 and epoch > 0 and epoch % val_test_frequency == 0:
val_nll, test_nll = do_evaluation()
log("[val/test epoch %04d] %.4f %.4f" %
(epoch, val_nll, test_nll))
# Testing.
print(f"Testing epoch {epoch}.")
dmm.eval()
mini_batch = next(iter(dataloader_test))
fig = test_minibatch(dmm, mini_batch, args)
fig.savefig(f'test_batch_{epoch}.png')
plt.close(fig)
# if 1:
# fig, _, _ = run_tsne(dmm, dataloader_test)
# fig.savefig(f'tsne_{epoch}.png')
# plt.close(fig)
print("Testing")
if 1:
dmm.eval()
mini_batch = next(iter(dataloader_test))
x, z, x_reconst = test_minibatch(dmm, mini_batch, args)
n_plots = 5
fig, axes = plt.subplots(nrows=n_plots, ncols=1)
for i in range(n_plots):
input = x[i, :].numpy().squeeze()
output = x_reconst[i, :]
axes[i].plot(range(input.shape[0]), input)
axes[i].plot(range(len(output)), output)
axes[i].grid()
fig.savefig(f'test_batch.png')
plt.close(fig)
if 1:
# t-SNE.
all_z_latents = []
for test_batch in dataloader_test:
# z_latents = minibatch_inference(dmm, test_batch)
# z_latents = encode_x_to_z(dmm, test_batch, sample_z_t=False)
x, z, x_reconst = test_minibatch(dmm,
test_batch,
args,
sample_z=True)
all_z_latents.append(z[:, x.shape[1] - 1, :])
# all_latents = torch.cat(all_z_latents, dim=0)
all_latents = np.concatenate(all_z_latents, axis=0)
# Run t-SNE with 2 output dimensions.
from sklearn.manifold import TSNE
model_tsne = TSNE(n_components=2, random_state=0)
# z_states = all_latents.detach().cpu().numpy()
z_states = all_latents
z_embed = model_tsne.fit_transform(z_states)
# Plot t-SNE embedding.
fig = plt.figure()
plt.scatter(z_embed[:, 0], z_embed[:, 1], s=10)
fig.savefig(f'tsne_test.png')
plt.close(fig)
return
# Show some samples surrounding a given point.
mini_batch = next(iter(dataloader_test))
# Use the inference network to determine the parameters of the latents.
z_loc, z_scale = minibatch_latent_parameters(dmm, mini_batch)
z = sample_latent_sequence(dmm, z_loc, z_scale)
most_recent_latents = latents[:, -1, :]
# Take
n_random_samples = 9
print('Finished')
def train():
parser = argparse.ArgumentParser(description='Train VAE.')
parser.add_argument('-c', '--config', default='train_config.json', help='Config file.')
args = parser.parse_args()
print(args)
c = json.load(open(args.config))
print(c)
pyro.clear_param_store()
# TODO: Move to config file.
lookback = 50
max_n_files = None
train_start_date = datetime.strptime(c['train_start_date'], '%Y/%m/%d')
train_end_date = datetime.strptime(c['train_end_date'], '%Y/%m/%d')
val_start_date = datetime.strptime(c['val_start_date'], '%Y/%m/%d')
val_end_date = datetime.strptime(c['val_end_date'], '%Y/%m/%d')
min_sequence_length_train = 2 * (c['series_length'] + lookback)
min_sequence_length_test = 2 * (c['series_length'] + lookback)
out_path = Path(c['out_dir'])
out_path.mkdir(exist_ok=True)
dataset_train = create_ticker_dataset(c['in_dir'], c['series_length'], lookback, min_sequence_length_train,
start_date=train_start_date, end_date=train_end_date,
normalised_returns=c['normalised_returns'], max_n_files=max_n_files)
dataset_val = create_ticker_dataset(c['in_dir'], c['series_length'], lookback, min_sequence_length_test,
start_date=val_start_date, end_date=val_end_date, fixed_start_date=True,
normalised_returns=c['normalised_returns'], max_n_files=max_n_files)
train_loader = DataLoader(dataset_train, batch_size=c['batch_size'], shuffle=True, num_workers=0, drop_last=True)
val_loader = DataLoader(dataset_val, batch_size=c['batch_size'], shuffle=False, num_workers=0, drop_last=True)
N_train_data = len(dataset_train)
N_val_data = len(dataset_val)
N_mini_batches = N_train_data // c['batch_size']
N_train_time_slices = c['batch_size'] * N_mini_batches
print(f'N_train_data: {N_train_data}, N_val_data: {N_val_data}')
# setup the VAE
vae = VAE(c['series_length'], z_dim=c['z_dim'], hidden_dims=c['hidden_dims'], use_cuda=c['cuda'])
# setup the optimizer
adam_args = {"lr": c['learning_rate']}
optimizer = Adam(adam_args)
# setup the inference algorithm
elbo = JitTrace_ELBO() if c['jit'] else Trace_ELBO()
svi = SVI(vae.model, vae.guide, optimizer, loss=elbo)
if c['checkpoint_load']:
checkpoint = torch.load(c['checkpoint_load'])
vae.load_state_dict(checkpoint['model_state_dict'])
# optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
train_elbo = []
val_elbo = []
# training loop
for epoch in range(c['n_epochs']):
# initialize loss accumulator
epoch_loss = 0.
# do a training epoch over each mini-batch x returned
# by the data loader
for batch in train_loader:
x = batch['series']
# if on GPU put mini-batch into CUDA memory
if c['cuda']:
x = x.cuda()
# do ELBO gradient and accumulate loss
epoch_loss += svi.step(x.float())
# report training diagnostics
normalizer_train = len(train_loader.dataset)
total_epoch_loss_train = epoch_loss / normalizer_train
train_elbo.append(total_epoch_loss_train)
print("[epoch %03d] average training loss: %.4f" % (epoch, total_epoch_loss_train))
torch.save({
'epoch': epoch,
'model_state_dict': vae.state_dict(),
# 'optimizer_state_dict': optimizer.state_dict(),
'train_loss': epoch_loss
}, out_path / c['checkpoint_save'].format(epoch))
if epoch % c['val_frequency'] == 0:
# initialize loss accumulator
val_loss = 0.
# compute the loss over the entire test set
for i, batch in enumerate(val_loader):
x = batch['series']
# if on GPU put mini-batch into CUDA memory
if c['cuda']:
x = x.cuda()
x = x.float()
# compute ELBO estimate and accumulate loss
val_loss += svi.evaluate_loss(x)
if i == 0:
# Visualise first batch.
x_reconst = vae.reconstruct_img(x)
x = x.cpu().numpy()
x_reconst = x_reconst.cpu().detach().numpy()
n = min(5, x.shape[0])
fig, axes = plt.subplots(n, 1, squeeze=False)
for s in range(n):
ax = axes[s, 0]
ax.plot(x[s])
ax.plot(x_reconst[s])
fig.savefig(out_path / f'val_{epoch:03d}.png')
plt.close(fig)
# report test diagnostics
normalizer_val = len(val_loader.dataset)
total_epoch_loss_val = val_loss / normalizer_val
val_elbo.append(total_epoch_loss_val)
print("[epoch %03d] average val loss: %.4f" % (epoch, total_epoch_loss_val))
# t-SNE.
all_z_latents = []
for batch in val_loader:
x = batch['series']
# z_latents = minibatch_inference(dmm, test_batch)
# z_latents = encode_x_to_z(dmm, test_batch, sample_z_t=False)
# x, z, x_reconst = test_minibatch(dmm, test_batch, args, sample_z=True)
if c['cuda']:
x = x.cuda()
z_loc, z_scale, z = vae.encode_x(x.float())
all_z_latents.append(z.cpu().numpy())
# all_latents = torch.cat(all_z_latents, dim=0)
all_latents = np.concatenate(all_z_latents, axis=0)
# Run t-SNE with 2 output dimensions.
from sklearn.manifold import TSNE
model_tsne = TSNE(n_components=2, random_state=0)
# z_states = all_latents.detach().cpu().numpy()
z_states = all_latents
z_embed = model_tsne.fit_transform(z_states)
# Plot t-SNE embedding.
fig = plt.figure()
plt.scatter(z_embed[:, 0], z_embed[:, 1], s=10)
fig.savefig(out_path / f'tsne_{epoch:03d}.png')
plt.close(fig)
print('Finished training.')
def test():
parser = argparse.ArgumentParser(description='Train VAE.')
parser.add_argument('-c', '--config', help='Config file.')
args = parser.parse_args()
print(args)
c = json.load(open(args.config))
print(c)
# clear param store
pyro.clear_param_store()
lookback = 50
test_start_date = datetime.strptime(
c['test_start_date'], '%Y/%m/%d') if c['test_start_date'] else None
test_end_date = datetime.strptime(
c['test_end_date'], '%Y/%m/%d') if c['test_end_date'] else None
min_sequence_length_test = 2 * (c['series_length'] + lookback)
max_n_files = None
out_path = Path(c['out_dir'])
out_path.mkdir(exist_ok=True)
dataset_test = create_ticker_dataset(
c['in_dir'],
c['series_length'],
lookback,
min_sequence_length_test,
start_date=test_start_date,
end_date=test_end_date,
fixed_start_date=True,
normalised_returns=c['normalised_returns'],
max_n_files=max_n_files)
test_loader = DataLoader(dataset_test,
batch_size=c['batch_size'],
shuffle=False,
num_workers=0,
drop_last=True)
N_test_data = len(dataset_test)
print(f'N_test_data: {N_test_data}')
# setup the VAE
vae = VAE(c['series_length'], z_dim=c['z_dim'], use_cuda=c['cuda'])
if c['checkpoint_load']:
checkpoint = torch.load(c['checkpoint_load'])
vae.load_state_dict(checkpoint['model_state_dict'])
# optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
if 1:
find_similar(vae, test_loader, c['cuda'])
# Visualise first batch.
batch = next(iter(test_loader))
x = batch['series']
if c['cuda']:
x = x.cuda()
x = x.float()
x_reconst = vae.reconstruct_img(x)
x = x.cpu().numpy()
x_reconst = x_reconst.cpu().detach().numpy()
n = min(5, x.shape[0])
fig, axes = plt.subplots(n, 1, squeeze=False)
for s in range(n):
ax = axes[s, 0]
ax.plot(x[s])
ax.plot(x_reconst[s])
fig.savefig(out_path / f'test_batch.png')
def main(args):
# clear param store
pyro.clear_param_store()
batch_size = 64
root_dir = r'D:\projects\trading\mlbootcamp\tickers'
series_length = 60
lookback = 50 # 160
input_dim = 1
train_start_date = datetime.date(2010, 1, 1)
train_end_date = datetime.date(2016, 1, 1)
test_start_date = train_end_date
test_end_date = datetime.date(2019, 1, 1)
min_sequence_length_train = 2 * (series_length + lookback)
min_sequence_length_test = 2 * (series_length + lookback)
max_n_files = None
load_path = 'out/checkpoint_0035.pt'
# setup MNIST data loaders
# train_loader, test_loader
# train_loader, test_loader = setup_data_loaders(MNIST, use_cuda=args.cuda, batch_size=256)
dataset_train = create_ticker_dataset(root_dir,
series_length,
lookback,
min_sequence_length_train,
start_date=train_start_date,
end_date=train_end_date,
max_n_files=max_n_files)
dataset_test = create_ticker_dataset(root_dir,
series_length,
lookback,
min_sequence_length_test,
start_date=test_start_date,
end_date=test_end_date,
fixed_start_date=True,
max_n_files=max_n_files)
train_loader = DataLoader(dataset_train,
batch_size=batch_size,
shuffle=True,
num_workers=0,
drop_last=True)
test_loader = DataLoader(dataset_test,
batch_size=batch_size,
shuffle=False,
num_workers=0,
drop_last=True)
N_train_data = len(dataset_train)
N_test_data = len(dataset_test)
N_mini_batches = N_train_data // batch_size
N_train_time_slices = batch_size * N_mini_batches
print(f'N_train_data: {N_train_data}, N_test_data: {N_test_data}')
# setup the VAE
vae = VAE(series_length, use_cuda=args.cuda)
# setup the optimizer
adam_args = {"lr": args.learning_rate}
optimizer = Adam(adam_args)
# setup the inference algorithm
elbo = JitTrace_ELBO() if args.jit else Trace_ELBO()
svi = SVI(vae.model, vae.guide, optimizer, loss=elbo)
if load_path:
checkpoint = torch.load(load_path)
vae.load_state_dict(checkpoint['model_state_dict'])
# optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
train_elbo = []
test_elbo = []
# training loop
for epoch in range(args.num_epochs):
# initialize loss accumulator
epoch_loss = 0.
# do a training epoch over each mini-batch x returned
# by the data loader
for x in train_loader:
# if on GPU put mini-batch into CUDA memory
if args.cuda:
x = x.cuda()
# do ELBO gradient and accumulate loss
epoch_loss += svi.step(x.float())
# report training diagnostics
normalizer_train = len(train_loader.dataset)
total_epoch_loss_train = epoch_loss / normalizer_train
train_elbo.append(total_epoch_loss_train)
print("[epoch %03d] average training loss: %.4f" %
(epoch, total_epoch_loss_train))
torch.save(
{
'epoch': epoch,
'model_state_dict': vae.state_dict(),
# 'optimizer_state_dict': optimizer.state_dict(),
'train_loss': epoch_loss
},
f'out/checkpoint_{epoch:04d}.pt')
if epoch % args.test_frequency == 0:
# initialize loss accumulator
test_loss = 0.
# compute the loss over the entire test set
for i, x in enumerate(test_loader):
# if on GPU put mini-batch into CUDA memory
if args.cuda:
x = x.cuda()
x = x.float()
# compute ELBO estimate and accumulate loss
test_loss += svi.evaluate_loss(x)
if i == 0:
# Visualise first batch.
x_reconst = vae.reconstruct_img(x)
x = x.cpu().numpy()
x_reconst = x_reconst.cpu().detach().numpy()
n = min(5, x.shape[0])
fig, axes = plt.subplots(n, 1, squeeze=False)
for s in range(n):
ax = axes[s, 0]
ax.plot(x[s])
ax.plot(x_reconst[s])
fig.savefig(f'out/val_{epoch:03d}.png')
# report test diagnostics
normalizer_test = len(test_loader.dataset)
total_epoch_loss_test = test_loss / normalizer_test
test_elbo.append(total_epoch_loss_test)
print("[epoch %03d] average test loss: %.4f" %
(epoch, total_epoch_loss_test))
# t-SNE.
all_z_latents = []
for x in test_loader:
# z_latents = minibatch_inference(dmm, test_batch)
# z_latents = encode_x_to_z(dmm, test_batch, sample_z_t=False)
# x, z, x_reconst = test_minibatch(dmm, test_batch, args, sample_z=True)
if args.cuda:
x = x.cuda()
z_loc, z_scale, z = vae.encode_x(x.float())
all_z_latents.append(z.cpu().numpy())
# all_latents = torch.cat(all_z_latents, dim=0)
all_latents = np.concatenate(all_z_latents, axis=0)
# Run t-SNE with 2 output dimensions.
from sklearn.manifold import TSNE
model_tsne = TSNE(n_components=2, random_state=0)
# z_states = all_latents.detach().cpu().numpy()
z_states = all_latents
z_embed = model_tsne.fit_transform(z_states)
# Plot t-SNE embedding.
fig = plt.figure()
plt.scatter(z_embed[:, 0], z_embed[:, 1], s=10)
fig.savefig(f'out/tsne_{epoch:03d}.png')
plt.close(fig)
if 1:
find_similar(vae, test_loader)
# Visualise first batch.
batch = next(iter(test_loader))
x = batch['series']
if args.cuda:
x = x.cuda()
x = x.float()
x_reconst = vae.reconstruct_img(x)
x = x.cpu().numpy()
x_reconst = x_reconst.cpu().detach().numpy()
n = min(5, x.shape[0])
fig, axes = plt.subplots(n, 1, squeeze=False)
for s in range(n):
ax = axes[s, 0]
ax.plot(x[s])
ax.plot(x_reconst[s])
fig.savefig(f'val_test.png')
return vae