def objective(options=None): # Initialize the best validation loss, which is the value to be minimized by the network best_val_loss = float("Inf") # Define hyperparameters lr = 0.001 dropout = 0.3 batch_size = 128 print(f"Learning rate: {lr}") print(f"Dropout: {dropout}") print(f"Batch size: {batch_size}") # Use CUDA if GPU is available, else CPU use_cuda = options["use_cuda"] and torch.cuda.is_available() device = torch.device("cuda" if use_cuda else "cpu") print(f"Using device {device}") # Obtain the MNIST train and validation loaders using a helper function train_loader, val_loader = get_mnist_dataloaders(options["data_path"], batch_size) # Initialize network model = Net(dropout=dropout).to(device) # Learning rate optimizer optimizer = optim.Adam(model.parameters(), lr=lr) scheduler = StepLR(optimizer, step_size=1, gamma=0.7) # Network training & validation loop for epoch in range(0, options["epochs"]): avg_train_loss = train( options, model, device, train_loader, optimizer, epoch ) avg_val_loss = validate(model, device, val_loader) if avg_val_loss <= best_val_loss: best_val_loss = avg_val_loss # Print intermediate validation & training loss print(f"Epoch {epoch + 1} of {options['epochs']} --- average train loss: {avg_train_loss} --- average validation loss: {avg_val_loss}") scheduler.step() # Return the best validation loss return best_val_loss
import model_use import preprocessing import training print('Loading dataset') data_texts, data_labels = datasets.load_test_data() print('Loading model') model, tokenizer = model_use.load_model() print('Preprocessing on dataset') data_inputs, _, _ = preprocessing.preprocess(data_texts, tokenizer=tokenizer) # Remove neutral data and labels from the csv file, since this is a binary classification model result = np.where(data_labels == 2)[0] data_inputs = np.delete(data_inputs, result, axis=0) data_labels = np.delete(data_labels, result, axis=0) print('Running validation') #K=2 in k fold cross validation. n_splits=2 results = training.validate(model, data_inputs, data_labels, num_epoch=20, n_splits=2) print(results) accuracies = [result[1][1] for result in results] print('Fold Accuracies:', ', '.join([str(a) for a in accuracies])) print('Average Accuracy:', sum(accuracies) / len(accuracies))
def train_and_validate(expected_numbers_of_worms_per_video, cluster_threshold=DEFAULT_CLUSTER_THRESHOLD, greyscale_threshold=DEFAULT_GREYSCALE_THRESHOLD): cluster_threshold, greyscale_threshold = training.train_using_monte_carlo_simulation(expected_numbers_of_worms_per_video["training"]) return training.validate(expected_numbers_of_worms_per_video["validation"], cluster_threshold, greyscale_threshold)
softmax_proteins = next(iter(train_loader))[:4] softmax_name = args.results_dir / Path("softmax.png") if args.anneal_learning_rates and args.plot_learning_rates: learning_rates = [] learning_rates_name = args.results_dir / Path("learning_rates.png") try: for epoch in range(1, args.epochs + 1): start_time = time.time() train_loss, train_metrics = train_epoch( epoch, model, optimizer, train_loader, args.log_interval, args.clip_grad_norm, args.clip_grad_value, scheduler) if args.val_ratio > 0: val_loss, val_metrics = validate(epoch, model, val_loader) loss_str = "Validation" loss_value_str = f"{val_loss:.5f}" val_str = f"{loss_str} loss: {loss_value_str} " improved = val_loss < best_loss else: loss_str = "Training" loss_value_str = f"{train_loss:.5f}" val_str = "" improved = train_loss < best_loss if improved: # If model improved, save the model model.save(model_save_name) print(
def importance(model, X_train, y_train, optimizer, criterion, epochs, device, val_dl, prop, writer=None, early_stopping=True, batch_size=64, recompute_epoch=0): # Compute initial importance values unique_labels = len(list(set(y_train))) importances = compute_samples_property(model, X_train, y_train, prop=prop, unique_labels=unique_labels, indices=False, verbose=True) # Initialization model.train() curr_epoch, size = 0, len(importances) rnd_indices = np.random.randint(0, len(X_train), batch_size) running_loss, running_corrects, total = 0, 0, 0 train_losses, train_accuracies, val_losses, val_accuracies = [], [], [], [] if recompute_epoch == 0: recompute_epoch = epochs # Simulate an epoch using len(X_train) = batch_size * batches while curr_epoch < epochs: if curr_epoch % recompute_epoch == 0 and curr_epoch != 0: importances = compute_samples_property(model, X_train, y_train, prop=prop, unique_labels=unique_labels, indices=False, verbose=True) seen_samples, running_loss = 0, 0 while seen_samples < len(X_train): if curr_epoch == 0: curr_indices = rnd_indices else: curr_indices = metropolis_hastings(importances, curr_indices) inputs, labels = X_train[curr_indices], y_train[curr_indices] inputs = np.moveaxis(inputs, source=-1, destination=1).astype(np.float32) inputs, labels = torch.Tensor(inputs), torch.LongTensor(labels) inputs, labels = inputs.to(device), labels.to(device) optimizer.zero_grad() outputs = model(inputs) loss = criterion(outputs, labels) loss.backward() optimizer.step() _, predicted = torch.max(outputs.data, 1) labels = labels.long() running_corrects += (predicted == labels).sum().item() running_loss += loss total += labels.size(0) seen_samples += len(curr_indices) # Train loss/accuracy train_loss = running_loss train_acc = running_corrects / total train_losses.append(train_loss) train_accuracies.append(train_acc) # Validation loss/accuracy val_loss, val_acc = validate(model, val_dl, criterion, device, flatten=False) val_losses.append(val_loss) val_accuracies.append(val_acc) curr_epoch += 1 print( '[ EPOCH {}: train_loss={} | train_acc={} | val_loss={} | val_acc={} ]' .format(curr_epoch, train_loss, train_acc, val_loss, val_acc))
opponent_value_function = None opponent_agent = BareAgent(opponent_policy, opponent_value_function) opponent_agents = train(player_agent_train, opponent_agent, 2001, 1000, 500) # player_value_function.save('SARSA_MAX_backward__UCB_vs_Self') print("ended training!") player_policy_val = GreedyPolicy() player_agent_val = BareAgent(player_policy_val, player_value_function) # opponent_policy = RandomPolicy() # opponent_value_function = None # opponent_agent = BareAgent(opponent_policy, opponent_value_function) # opponent_agents = [opponent_agent] validate(player_agent_val, opponent_agents, 501) # ## ------------------------------- #### # # # print("ended validation") ## ------------------------------- #### # # opponent_policy = RandomPolicy() # opponent_policy = GreedyPolicy() # opponent_value_function = LookupTable() # opponent_value_function.load('SARSA_MAX_backward__UCB_vs_Self') # opponent_agent = BareAgent(opponent_policy, opponent_value_function) # # while True: # play_human(opponent_agent, True)
train_loss_count += 1 print_seqs_count += batch_size if print_seqs_count >= print_every_samples: process = psutil.Process(os.getpid()) print(f'Progress: {100 * seqs_processed / train_seqs_per_epoch:6.3f}% of epoch. Total time: {readable_time(time.time() - print_seqs_overall_time):>7s}. Iteration time: {readable_time(time.time() - print_seqs_iteration_time):>7s} CPU Memory: {process.memory_info().rss / (1024**3)} GiB') print_seqs_iteration_time = time.time() print_seqs_count = 0 if seqs_processed >= train_seqs_per_epoch: epoch += 1 train_loss = acc_train_loss / train_loss_count seqs_processed = 0 acc_train_loss = 0 train_loss_count = 0 val_loss, _ = validate(epoch, model, val_loader) # If save the latest model if args.multi_gpu: model.module.save(model_save_name_latest) else: model.save(model_save_name_latest) improved = val_loss < best_val_loss if improved: # If model improved, save the model if args.multi_gpu: model.module.save(model_save_name) else: model.save(model_save_name)
def compare_approaches(save_fig=True): # Setup date = datetime.now().strftime("%m-%d_%H-%M-%S") toy = ToyExperiment(n_features=2, hidden_dim=5, n_classes=2, n_samples=500, theta=90, N_EPOCHS=20, convert_y=False) ss = StandardScaler() X_train_sc = ss.fit_transform(toy.X_train) X_train_noisy_sc = ss.fit_transform(toy.X_train_noisy) X_valid_noisy_sc = ss.fit_transform(toy.X_valid_noisy) # Compute loss and accuracy of clean model on noisy dataset train_dl = get_data_loader(X_train_noisy_sc, toy.y_train) valid_dl = get_data_loader(X_valid_noisy_sc, toy.y_valid) train_loss_clean, train_acc_clean = validate(toy.model_clean, train_dl, toy.criterion, device, flatten=True) valid_loss_clean, valid_acc_clean = validate(toy.model_clean, valid_dl, toy.criterion, device, flatten=True) # Percentage to keep perc = 0.5 n_idx = int(len(X_train_sc) * perc) # Leave one out loo_losses = get_leave_one_out_losses(toy, dset='valid') loo_idx = np.argsort(loo_losses) loo_idx = loo_idx[::-1][:n_idx] # leave_one_out_idx = np.random.choice(len(X_train_noisy_sc), n_idx, replace=False) # Entropy entropy_idx = compute_samples_property(toy.model_clean, toy.X_train_noisy, toy.y_train, prop='entropy', flatten=True, indices=True, unique_labels=[0, 1]) entropy_idx = entropy_idx[::-1][:n_idx] # Hessian hessian_losses = instance_selection(model=toy.model_clean, X_train=X_train_noisy_sc, y_train=toy.y_train, sparse=False, X_valid=X_valid_noisy_sc, y_valid=toy.y_valid, criterion=toy.criterion, treshold=False, return_influences=True) hessian_idx = np.argsort(hessian_losses)[:n_idx] # Jacobian jacobian_losses = instance_selection_no_hessian(model=toy.model_clean, X_train=X_train_noisy_sc, y_train=toy.y_train, X_valid=X_valid_noisy_sc, y_valid=toy.y_valid, criterion=toy.criterion, flatten=True, return_influences=True) jacobian_idx = np.argsort(jacobian_losses)[:n_idx] # Training derivatives train_derivatives = instance_selection_train_derivatives( model=toy.model_clean, X=X_train_noisy_sc, y=toy.y_train, criterion=toy.criterion, flatten=True) train_derivatives_idx = np.argsort(train_derivatives)[:n_idx] # n_idx = min(len(hessian_idx), len(jacobian_idx)) # Random random_idx = np.random.choice(len(X_train_noisy_sc), n_idx, replace=False) # Generate scatterplot methods = [(random_idx, 'random'), (loo_idx, 'leave one out'), (entropy_idx, 'entropy'), (hessian_idx, 'hessian'), (jacobian_idx, 'jacobian'), [train_derivatives_idx, 'train derivatives norms']] create_scatterplot(X_train_sc, X_train_noisy_sc, toy.y_train, X_valid_noisy_sc, toy.y_valid, methods, n_idx, date=date) # Print common selected indices between loo and jacobian common = len(list(set(loo_idx) & set(jacobian_idx))) print('Common indices LOO and Jacobian: {}/{} | {:.2f}%'.format( common, n_idx, common / n_idx * 100)) # Train plt.figure(figsize=(15, 5)) for idx, name in tqdm(methods, desc='Training'): model_ = copy.deepcopy(toy.model_clean) X_train_ = X_train_noisy_sc[idx][:n_idx] y_train_ = toy.y_train[idx][:n_idx] optimizer = torch.optim.SGD(model_.parameters(), lr=0.01) train_dl_ = get_data_loader(X_train_, y_train_) train_losses_, train_accs_, val_accs_, val_losses_, epoch_, _ = train( model=model_, train_loader=train_dl_, val_loader=toy.valid_noisy_dl, test_loader=toy.test_noisy_dl, optimizer=optimizer, criterion=toy.criterion, device=device, epochs=N_EPOCHS, early_stopping=False, flatten=True, verbose=False) # Make accuracies and losses starting from the same intial point train_losses_.insert(0, train_loss_clean) train_accs_.insert(0, train_acc_clean) val_losses_.insert(0, valid_loss_clean) val_accs_.insert(0, valid_acc_clean) # Plot plt.subplot(1, 2, 1) plt.plot(range(N_EPOCHS + 1), train_accs_, label=name) plt.xlabel('Epochs', fontsize=16) plt.ylabel('Accuracy', fontsize=16) plt.title('Train\n') plt.legend(loc='best') plt.subplot(1, 2, 2) plt.plot(range(N_EPOCHS + 1), val_accs_, label=name) plt.xlabel('Epochs', fontsize=16) plt.ylabel('Accuracy', fontsize=16) plt.title('Validation\n') plt.legend(loc='best') if save_fig: plt.savefig(ROOT_DIR + '/toy-model/images/compare_{}_accuracy.png'.format(date)) plt.show()
def run(args): print('\nSettings: \n', args, '\n') args.model_signature = str(dt.datetime.now())[0:19].replace(' ', '_') args.model_signature = args.model_signature.replace(':', '_') ########## Find GPUs (gpu_config, n_gpu_used) = set_gpus(args.n_gpu) ########## Data, model, and optimizer setup mnist = MNIST(args) x = tf.placeholder(tf.float32, [None, 28, 28, 1]) if args.model == 'hvae': if not args.K: raise ValueError('Must set number of flow steps when using HVAE') elif not args.temp_method: raise ValueError('Must set tempering method when using HVAE') model = HVAE(args, mnist.avg_logit) elif args.model == 'cnn': model = VAE(args, mnist.avg_logit) else: raise ValueError('Invalid model choice') elbo = model.get_elbo(x, args) nll = model.get_nll(x, args) optimizer = AdamaxOptimizer(learning_rate=args.learn_rate, eps=args.adamax_eps) opt_step = optimizer.minimize(-elbo) ########## Tensorflow and saver setup sess = tf.Session(config=gpu_config) sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() savepath = os.path.join(args.checkpoint_dir, args.model_signature, 'model.ckpt') if not os.path.exists(args.checkpoint_dir): os.makedirs(args.checkpoint_dir) ########## Test that GPU memory is sufficient if n_gpu_used > 0: try: x_test = mnist.next_test_batch() (t_e, t_n) = sess.run((elbo, nll), {x: x_test}) mnist.batch_idx_test = 0 # Reset batch counter if it works except: raise MemoryError(""" Likely insufficient GPU memory Reduce test batch by lowering the -tbs parameter """) ########## Training Loop train_elbo_hist = [] val_elbo_hist = [] # For early stopping best_elbo = -np.inf es_epochs = 0 epoch = 0 train_times = [] for epoch in range(1, args.epochs + 1): t0 = time.time() train_elbo = train(epoch, mnist, opt_step, elbo, x, args, sess) train_elbo_hist.append(train_elbo) train_times.append(time.time() - t0) print('One epoch took {:.2f} seconds'.format(time.time() - t0)) val_elbo = validate(mnist, elbo, x, sess) val_elbo_hist.append(val_elbo) if val_elbo > best_elbo: # Save the model that currently generalizes best es_epochs = 0 best_elbo = val_elbo saver.save(sess, savepath) best_model_epoch = epoch elif args.early_stopping_epochs > 0: es_epochs += 1 if es_epochs >= args.early_stopping_epochs: print('***** STOPPING EARLY ON EPOCH {} of {} *****'.format( epoch, args.epochs)) break print('--> Early stopping: {}/{} (Best ELBO: {:.4f})'.format( es_epochs, args.early_stopping_epochs, best_elbo)) print('\t Current val ELBO: {:.4f}\n'.format(val_elbo)) if np.isnan(val_elbo): raise ValueError('NaN encountered!') train_times = np.array(train_times) mean_time = np.mean(train_times) std_time = np.std(train_times) print('Average train time per epoch: {:.2f} +/- {:.2f}'.format( mean_time, std_time)) ########## Evaluation # Restore the best-performing model saver.restore(sess, savepath) test_elbos = np.zeros(args.n_nll_runs) test_nlls = np.zeros(args.n_nll_runs) for i in range(args.n_nll_runs): print('\n---- Test run {} of {} ----\n'.format(i + 1, args.n_nll_runs)) (test_elbos[i], test_nlls[i]) = evaluate(mnist, elbo, nll, x, args, sess) mean_elbo = np.mean(test_elbos) std_elbo = np.std(test_elbos) mean_nll = np.mean(test_nlls) std_nll = np.std(test_nlls) print('\nTest ELBO: {:.2f} +/- {:.2f}'.format(mean_elbo, std_elbo)) print('Test NLL: {:.2f} +/- {:.2f}'.format(mean_nll, std_nll)) ########## Logging, Saving, and Plotting with open(args.logfile, 'a') as ff: print('----------------- Test ID {} -----------------'.format( args.model_signature), file=ff) print(args, file=ff) print('Stopped after {} epochs'.format(epoch), file=ff) print('Best model from epoch {}'.format(best_model_epoch), file=ff) print('Average train time per epoch: {:.2f} +/- {:.2f}'.format( mean_time, std_time), file=ff) print('FINAL VALIDATION ELBO: {:.2f}'.format(val_elbo_hist[-1]), file=ff) print('Test ELBO: {:.2f} +/- {:.2f}'.format(mean_elbo, std_elbo), file=ff) print('Test NLL: {:.2f} +/- {:.2f}\n'.format(mean_nll, std_nll), file=ff) if not os.path.exists(args.pickle_dir): os.makedirs(args.pickle_dir) train_dict = { 'train_elbo': train_elbo_hist, 'val_elbo': val_elbo_hist, 'args': args } pickle.dump( train_dict, open(os.path.join(args.pickle_dir, args.model_signature + '.p'), 'wb')) if not os.path.exists(args.plot_dir): os.makedirs(args.plot_dir) tf_gen_samples = model.get_samples(args) np_gen_samples = sess.run(tf_gen_samples) plot_digit_samples(np_gen_samples, args) plot_training_curve(train_elbo_hist, val_elbo_hist, args) ########## Email notification upon test completion try: msg_text = """Test completed for ID {0}. Parameters: {1} Test ELBO: {2:.2f} +/- {3:.2f} Test NLL: {4:.2f} +/- {5:.2f} """.format(args.model_signature, args, mean_elbo, std_elbo, mean_nll, std_nll) msg = MIMEText(msg_text) msg['Subject'] = 'Test ID {0} Complete'.format(args.model_signature) msg['To'] = args.receiver msg['From'] = args.sender s = smtplib.SMTP('localhost') s.sendmail(args.sender, [args.receiver], msg.as_string()) s.quit() except: print('Unable to send email from sender {0} to receiver {1}'.format( args.sender, args.receiver))