def test(): parser = argparse.ArgumentParser(description='Train VAE.') parser.add_argument('-c', '--config', help='Config file.') args = parser.parse_args() print(args) c = json.load(open(args.config)) print(c) # clear param store pyro.clear_param_store() # batch_size = 64 # root_dir = r'D:\projects\trading\mlbootcamp\tickers' # series_length = 60 lookback = 50 # 160 input_dim = 1 test_start_date = datetime.strptime( c['test_start_date'], '%Y/%m/%d') if c['test_start_date'] else None test_end_date = datetime.strptime( c['test_end_date'], '%Y/%m/%d') if c['test_end_date'] else None min_sequence_length_test = 2 * (c['series_length'] + lookback) max_n_files = None out_path = Path(c['out_dir']) out_path.mkdir(exist_ok=True) # load_path = 'out_saved/checkpoint_0035.pt' dataset_test = create_ticker_dataset( c['in_dir'], c['series_length'], lookback, min_sequence_length_test, start_date=test_start_date, end_date=test_end_date, fixed_start_date=True, normalised_returns=c['normalised_returns'], max_n_files=max_n_files) test_loader = DataLoader(dataset_test, batch_size=c['batch_size'], shuffle=False, num_workers=0, drop_last=True) # N_train_data = len(dataset_train) N_test_data = len(dataset_test) # N_mini_batches = N_train_data // c['batch_size'] # N_train_time_slices = c['batch_size'] * N_mini_batches print(f'N_test_data: {N_test_data}') # setup the VAE vae = VAE(c['series_length'], z_dim=c['z_dim'], use_cuda=c['cuda']) # setup the optimizer # adam_args = {"lr": args.learning_rate} # optimizer = Adam(adam_args) # setup the inference algorithm # elbo = JitTrace_ELBO() if args.jit else Trace_ELBO() # svi = SVI(vae.model, vae.guide, optimizer, loss=elbo) if c['checkpoint_load']: checkpoint = torch.load(c['checkpoint_load']) vae.load_state_dict(checkpoint['model_state_dict']) # optimizer.load_state_dict(checkpoint['optimizer_state_dict']) if 1: find_similar(vae, test_loader, c['cuda']) # Visualise first batch. batch = next(iter(test_loader)) x = batch['series'] if c['cuda']: x = x.cuda() x = x.float() x_reconst = vae.reconstruct_img(x) x = x.cpu().numpy() x_reconst = x_reconst.cpu().detach().numpy() n = min(5, x.shape[0]) fig, axes = plt.subplots(n, 1, squeeze=False) for s in range(n): ax = axes[s, 0] ax.plot(x[s]) ax.plot(x_reconst[s]) fig.savefig(out_path / f'test_batch.png')
def main(args): # setup logging log = get_logger(args.log) log(args) root_dir = r'D:\projects\trading\mlbootcamp\tickers' series_length = 60 lookback = 50 #160 input_dim = 1 train_start_date = datetime.date(2010, 1, 1) train_end_date = datetime.date(2016, 1, 1) test_start_date = train_end_date test_end_date = datetime.date(2019, 1, 1) min_sequence_length_train = 2 * (series_length + lookback) min_sequence_length_test = 2 * (series_length + lookback) dataset_train = create_ticker_dataset(root_dir, series_length, lookback, min_sequence_length_train, start_date=train_start_date, end_date=train_end_date) dataset_test = create_ticker_dataset(root_dir, series_length, lookback, min_sequence_length_test, start_date=test_start_date, end_date=test_end_date) dataloader_train = DataLoader(dataset_train, batch_size=args.mini_batch_size, shuffle=True, num_workers=0, drop_last=True) dataloader_test = DataLoader(dataset_test, batch_size=args.mini_batch_size, shuffle=False, num_workers=0, drop_last=True) N_train_data = len(dataset_train) N_test_data = len(dataset_test) N_mini_batches = N_train_data // args.mini_batch_size N_train_time_slices = args.mini_batch_size * N_mini_batches print(f'N_train_data: {N_train_data}, N_test_data: {N_test_data}') # how often we do validation/test evaluation during training val_test_frequency = 50 # the number of samples we use to do the evaluation n_eval_samples = 1 # instantiate the dmm dmm = DMM(input_dim=input_dim, rnn_dropout_rate=args.rnn_dropout_rate, num_iafs=args.num_iafs, iaf_dim=args.iaf_dim, use_cuda=args.cuda) # setup optimizer adam_params = { "lr": args.learning_rate, "betas": (args.beta1, args.beta2), "clip_norm": args.clip_norm, "lrd": args.lr_decay, "weight_decay": args.weight_decay } adam = ClippedAdam(adam_params) # setup inference algorithm elbo = JitTrace_ELBO() if args.jit else Trace_ELBO() svi = SVI(dmm.model, dmm.guide, adam, loss=elbo) # if checkpoint files provided, load model and optimizer states from disk before we start training if args.load_opt != '' and args.load_model != '': load_checkpoint(dmm, adam, log) ################# # TRAINING LOOP # ################# times = [time.time()] for epoch in range(args.num_epochs): print(f'Starting epoch {epoch}.') # accumulator for our estimate of the negative log likelihood (or rather -elbo) for this epoch epoch_nll = 0.0 # prepare mini-batch subsampling indices for this epoch shuffled_indices = torch.randperm(N_train_data) # process each mini-batch; this is where we take gradient steps dmm.train() for which_mini_batch in range(N_mini_batches): print( f'Epoch {epoch} of {args.num_epochs}, Batch {which_mini_batch} of {N_mini_batches}.' ) mini_batch = next(iter(dataloader_train)) epoch_nll += process_minibatch(svi, epoch, mini_batch, N_mini_batches, which_mini_batch, shuffled_indices) # if specified, save model and optimizer states to disk every checkpoint_freq epochs if 1: #args.checkpoint_freq > 0 and epoch > 0 and epoch % args.checkpoint_freq == 0: save_checkpoint(dmm, adam, log) # report training diagnostics times.append(time.time()) epoch_time = times[-1] - times[-2] log("[training epoch %04d] %.4f \t\t\t\t(dt = %.3f sec)" % (epoch, epoch_nll / N_train_time_slices, epoch_time)) # do evaluation on test and validation data and report results if val_test_frequency > 0 and epoch > 0 and epoch % val_test_frequency == 0: val_nll, test_nll = do_evaluation() log("[val/test epoch %04d] %.4f %.4f" % (epoch, val_nll, test_nll)) # Testing. print(f"Testing epoch {epoch}.") dmm.eval() mini_batch = next(iter(dataloader_test)) fig = test_minibatch(dmm, mini_batch, args) fig.savefig(f'test_batch_{epoch}.png') plt.close(fig) # if 1: # fig, _, _ = run_tsne(dmm, dataloader_test) # fig.savefig(f'tsne_{epoch}.png') # plt.close(fig) print("Testing") if 1: dmm.eval() mini_batch = next(iter(dataloader_test)) x, z, x_reconst = test_minibatch(dmm, mini_batch, args) n_plots = 5 fig, axes = plt.subplots(nrows=n_plots, ncols=1) for i in range(n_plots): input = x[i, :].numpy().squeeze() output = x_reconst[i, :] axes[i].plot(range(input.shape[0]), input) axes[i].plot(range(len(output)), output) axes[i].grid() fig.savefig(f'test_batch.png') plt.close(fig) if 1: # t-SNE. all_z_latents = [] for test_batch in dataloader_test: # z_latents = minibatch_inference(dmm, test_batch) # z_latents = encode_x_to_z(dmm, test_batch, sample_z_t=False) x, z, x_reconst = test_minibatch(dmm, test_batch, args, sample_z=True) all_z_latents.append(z[:, x.shape[1] - 1, :]) # all_latents = torch.cat(all_z_latents, dim=0) all_latents = np.concatenate(all_z_latents, axis=0) # Run t-SNE with 2 output dimensions. from sklearn.manifold import TSNE model_tsne = TSNE(n_components=2, random_state=0) # z_states = all_latents.detach().cpu().numpy() z_states = all_latents z_embed = model_tsne.fit_transform(z_states) # Plot t-SNE embedding. fig = plt.figure() plt.scatter(z_embed[:, 0], z_embed[:, 1], s=10) fig.savefig(f'tsne_test.png') plt.close(fig) return # Show some samples surrounding a given point. mini_batch = next(iter(dataloader_test)) # Use the inference network to determine the parameters of the latents. z_loc, z_scale = minibatch_latent_parameters(dmm, mini_batch) z = sample_latent_sequence(dmm, z_loc, z_scale) most_recent_latents = latents[:, -1, :] # Take n_random_samples = 9 print('Finished')
def train(): parser = argparse.ArgumentParser(description='Train VAE.') parser.add_argument('-c', '--config', default='train_config.json', help='Config file.') args = parser.parse_args() print(args) c = json.load(open(args.config)) print(c) pyro.clear_param_store() # TODO: Move to config file. lookback = 50 max_n_files = None train_start_date = datetime.strptime(c['train_start_date'], '%Y/%m/%d') train_end_date = datetime.strptime(c['train_end_date'], '%Y/%m/%d') val_start_date = datetime.strptime(c['val_start_date'], '%Y/%m/%d') val_end_date = datetime.strptime(c['val_end_date'], '%Y/%m/%d') min_sequence_length_train = 2 * (c['series_length'] + lookback) min_sequence_length_test = 2 * (c['series_length'] + lookback) out_path = Path(c['out_dir']) out_path.mkdir(exist_ok=True) dataset_train = create_ticker_dataset(c['in_dir'], c['series_length'], lookback, min_sequence_length_train, start_date=train_start_date, end_date=train_end_date, normalised_returns=c['normalised_returns'], max_n_files=max_n_files) dataset_val = create_ticker_dataset(c['in_dir'], c['series_length'], lookback, min_sequence_length_test, start_date=val_start_date, end_date=val_end_date, fixed_start_date=True, normalised_returns=c['normalised_returns'], max_n_files=max_n_files) train_loader = DataLoader(dataset_train, batch_size=c['batch_size'], shuffle=True, num_workers=0, drop_last=True) val_loader = DataLoader(dataset_val, batch_size=c['batch_size'], shuffle=False, num_workers=0, drop_last=True) N_train_data = len(dataset_train) N_val_data = len(dataset_val) N_mini_batches = N_train_data // c['batch_size'] N_train_time_slices = c['batch_size'] * N_mini_batches print(f'N_train_data: {N_train_data}, N_val_data: {N_val_data}') # setup the VAE vae = VAE(c['series_length'], z_dim=c['z_dim'], hidden_dims=c['hidden_dims'], use_cuda=c['cuda']) # setup the optimizer adam_args = {"lr": c['learning_rate']} optimizer = Adam(adam_args) # setup the inference algorithm elbo = JitTrace_ELBO() if c['jit'] else Trace_ELBO() svi = SVI(vae.model, vae.guide, optimizer, loss=elbo) if c['checkpoint_load']: checkpoint = torch.load(c['checkpoint_load']) vae.load_state_dict(checkpoint['model_state_dict']) # optimizer.load_state_dict(checkpoint['optimizer_state_dict']) train_elbo = [] val_elbo = [] # training loop for epoch in range(c['n_epochs']): # initialize loss accumulator epoch_loss = 0. # do a training epoch over each mini-batch x returned # by the data loader for batch in train_loader: x = batch['series'] # if on GPU put mini-batch into CUDA memory if c['cuda']: x = x.cuda() # do ELBO gradient and accumulate loss epoch_loss += svi.step(x.float()) # report training diagnostics normalizer_train = len(train_loader.dataset) total_epoch_loss_train = epoch_loss / normalizer_train train_elbo.append(total_epoch_loss_train) print("[epoch %03d] average training loss: %.4f" % (epoch, total_epoch_loss_train)) torch.save({ 'epoch': epoch, 'model_state_dict': vae.state_dict(), # 'optimizer_state_dict': optimizer.state_dict(), 'train_loss': epoch_loss }, out_path / c['checkpoint_save'].format(epoch)) if epoch % c['val_frequency'] == 0: # initialize loss accumulator val_loss = 0. # compute the loss over the entire test set for i, batch in enumerate(val_loader): x = batch['series'] # if on GPU put mini-batch into CUDA memory if c['cuda']: x = x.cuda() x = x.float() # compute ELBO estimate and accumulate loss val_loss += svi.evaluate_loss(x) if i == 0: # Visualise first batch. x_reconst = vae.reconstruct_img(x) x = x.cpu().numpy() x_reconst = x_reconst.cpu().detach().numpy() n = min(5, x.shape[0]) fig, axes = plt.subplots(n, 1, squeeze=False) for s in range(n): ax = axes[s, 0] ax.plot(x[s]) ax.plot(x_reconst[s]) fig.savefig(out_path / f'val_{epoch:03d}.png') plt.close(fig) # report test diagnostics normalizer_val = len(val_loader.dataset) total_epoch_loss_val = val_loss / normalizer_val val_elbo.append(total_epoch_loss_val) print("[epoch %03d] average val loss: %.4f" % (epoch, total_epoch_loss_val)) # t-SNE. all_z_latents = [] for batch in val_loader: x = batch['series'] # z_latents = minibatch_inference(dmm, test_batch) # z_latents = encode_x_to_z(dmm, test_batch, sample_z_t=False) # x, z, x_reconst = test_minibatch(dmm, test_batch, args, sample_z=True) if c['cuda']: x = x.cuda() z_loc, z_scale, z = vae.encode_x(x.float()) all_z_latents.append(z.cpu().numpy()) # all_latents = torch.cat(all_z_latents, dim=0) all_latents = np.concatenate(all_z_latents, axis=0) # Run t-SNE with 2 output dimensions. from sklearn.manifold import TSNE model_tsne = TSNE(n_components=2, random_state=0) # z_states = all_latents.detach().cpu().numpy() z_states = all_latents z_embed = model_tsne.fit_transform(z_states) # Plot t-SNE embedding. fig = plt.figure() plt.scatter(z_embed[:, 0], z_embed[:, 1], s=10) fig.savefig(out_path / f'tsne_{epoch:03d}.png') plt.close(fig) print('Finished training.')
def test(): parser = argparse.ArgumentParser(description='Train VAE.') parser.add_argument('-c', '--config', help='Config file.') args = parser.parse_args() print(args) c = json.load(open(args.config)) print(c) # clear param store pyro.clear_param_store() lookback = 50 test_start_date = datetime.strptime( c['test_start_date'], '%Y/%m/%d') if c['test_start_date'] else None test_end_date = datetime.strptime( c['test_end_date'], '%Y/%m/%d') if c['test_end_date'] else None min_sequence_length_test = 2 * (c['series_length'] + lookback) max_n_files = None out_path = Path(c['out_dir']) out_path.mkdir(exist_ok=True) dataset_test = create_ticker_dataset( c['in_dir'], c['series_length'], lookback, min_sequence_length_test, start_date=test_start_date, end_date=test_end_date, fixed_start_date=True, normalised_returns=c['normalised_returns'], max_n_files=max_n_files) test_loader = DataLoader(dataset_test, batch_size=c['batch_size'], shuffle=False, num_workers=0, drop_last=True) N_test_data = len(dataset_test) print(f'N_test_data: {N_test_data}') # setup the VAE vae = VAE(c['series_length'], z_dim=c['z_dim'], use_cuda=c['cuda']) if c['checkpoint_load']: checkpoint = torch.load(c['checkpoint_load']) vae.load_state_dict(checkpoint['model_state_dict']) # optimizer.load_state_dict(checkpoint['optimizer_state_dict']) if 1: find_similar(vae, test_loader, c['cuda']) # Visualise first batch. batch = next(iter(test_loader)) x = batch['series'] if c['cuda']: x = x.cuda() x = x.float() x_reconst = vae.reconstruct_img(x) x = x.cpu().numpy() x_reconst = x_reconst.cpu().detach().numpy() n = min(5, x.shape[0]) fig, axes = plt.subplots(n, 1, squeeze=False) for s in range(n): ax = axes[s, 0] ax.plot(x[s]) ax.plot(x_reconst[s]) fig.savefig(out_path / f'test_batch.png')
def main(args): # clear param store pyro.clear_param_store() batch_size = 64 root_dir = r'D:\projects\trading\mlbootcamp\tickers' series_length = 60 lookback = 50 # 160 input_dim = 1 train_start_date = datetime.date(2010, 1, 1) train_end_date = datetime.date(2016, 1, 1) test_start_date = train_end_date test_end_date = datetime.date(2019, 1, 1) min_sequence_length_train = 2 * (series_length + lookback) min_sequence_length_test = 2 * (series_length + lookback) max_n_files = None load_path = 'out/checkpoint_0035.pt' # setup MNIST data loaders # train_loader, test_loader # train_loader, test_loader = setup_data_loaders(MNIST, use_cuda=args.cuda, batch_size=256) dataset_train = create_ticker_dataset(root_dir, series_length, lookback, min_sequence_length_train, start_date=train_start_date, end_date=train_end_date, max_n_files=max_n_files) dataset_test = create_ticker_dataset(root_dir, series_length, lookback, min_sequence_length_test, start_date=test_start_date, end_date=test_end_date, fixed_start_date=True, max_n_files=max_n_files) train_loader = DataLoader(dataset_train, batch_size=batch_size, shuffle=True, num_workers=0, drop_last=True) test_loader = DataLoader(dataset_test, batch_size=batch_size, shuffle=False, num_workers=0, drop_last=True) N_train_data = len(dataset_train) N_test_data = len(dataset_test) N_mini_batches = N_train_data // batch_size N_train_time_slices = batch_size * N_mini_batches print(f'N_train_data: {N_train_data}, N_test_data: {N_test_data}') # setup the VAE vae = VAE(series_length, use_cuda=args.cuda) # setup the optimizer adam_args = {"lr": args.learning_rate} optimizer = Adam(adam_args) # setup the inference algorithm elbo = JitTrace_ELBO() if args.jit else Trace_ELBO() svi = SVI(vae.model, vae.guide, optimizer, loss=elbo) if load_path: checkpoint = torch.load(load_path) vae.load_state_dict(checkpoint['model_state_dict']) # optimizer.load_state_dict(checkpoint['optimizer_state_dict']) train_elbo = [] test_elbo = [] # training loop for epoch in range(args.num_epochs): # initialize loss accumulator epoch_loss = 0. # do a training epoch over each mini-batch x returned # by the data loader for x in train_loader: # if on GPU put mini-batch into CUDA memory if args.cuda: x = x.cuda() # do ELBO gradient and accumulate loss epoch_loss += svi.step(x.float()) # report training diagnostics normalizer_train = len(train_loader.dataset) total_epoch_loss_train = epoch_loss / normalizer_train train_elbo.append(total_epoch_loss_train) print("[epoch %03d] average training loss: %.4f" % (epoch, total_epoch_loss_train)) torch.save( { 'epoch': epoch, 'model_state_dict': vae.state_dict(), # 'optimizer_state_dict': optimizer.state_dict(), 'train_loss': epoch_loss }, f'out/checkpoint_{epoch:04d}.pt') if epoch % args.test_frequency == 0: # initialize loss accumulator test_loss = 0. # compute the loss over the entire test set for i, x in enumerate(test_loader): # if on GPU put mini-batch into CUDA memory if args.cuda: x = x.cuda() x = x.float() # compute ELBO estimate and accumulate loss test_loss += svi.evaluate_loss(x) if i == 0: # Visualise first batch. x_reconst = vae.reconstruct_img(x) x = x.cpu().numpy() x_reconst = x_reconst.cpu().detach().numpy() n = min(5, x.shape[0]) fig, axes = plt.subplots(n, 1, squeeze=False) for s in range(n): ax = axes[s, 0] ax.plot(x[s]) ax.plot(x_reconst[s]) fig.savefig(f'out/val_{epoch:03d}.png') # report test diagnostics normalizer_test = len(test_loader.dataset) total_epoch_loss_test = test_loss / normalizer_test test_elbo.append(total_epoch_loss_test) print("[epoch %03d] average test loss: %.4f" % (epoch, total_epoch_loss_test)) # t-SNE. all_z_latents = [] for x in test_loader: # z_latents = minibatch_inference(dmm, test_batch) # z_latents = encode_x_to_z(dmm, test_batch, sample_z_t=False) # x, z, x_reconst = test_minibatch(dmm, test_batch, args, sample_z=True) if args.cuda: x = x.cuda() z_loc, z_scale, z = vae.encode_x(x.float()) all_z_latents.append(z.cpu().numpy()) # all_latents = torch.cat(all_z_latents, dim=0) all_latents = np.concatenate(all_z_latents, axis=0) # Run t-SNE with 2 output dimensions. from sklearn.manifold import TSNE model_tsne = TSNE(n_components=2, random_state=0) # z_states = all_latents.detach().cpu().numpy() z_states = all_latents z_embed = model_tsne.fit_transform(z_states) # Plot t-SNE embedding. fig = plt.figure() plt.scatter(z_embed[:, 0], z_embed[:, 1], s=10) fig.savefig(f'out/tsne_{epoch:03d}.png') plt.close(fig) if 1: find_similar(vae, test_loader) # Visualise first batch. batch = next(iter(test_loader)) x = batch['series'] if args.cuda: x = x.cuda() x = x.float() x_reconst = vae.reconstruct_img(x) x = x.cpu().numpy() x_reconst = x_reconst.cpu().detach().numpy() n = min(5, x.shape[0]) fig, axes = plt.subplots(n, 1, squeeze=False) for s in range(n): ax = axes[s, 0] ax.plot(x[s]) ax.plot(x_reconst[s]) fig.savefig(f'val_test.png') return vae