def objective(options=None):
# Initialize the best validation loss, which is the value to be minimized by the network
best_val_loss = float("Inf")
# Define hyperparameters
lr = 0.001
dropout = 0.3
batch_size = 128
print(f"Learning rate: {lr}")
print(f"Dropout: {dropout}")
print(f"Batch size: {batch_size}")
# Use CUDA if GPU is available, else CPU
use_cuda = options["use_cuda"] and torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")
print(f"Using device {device}")
# Obtain the MNIST train and validation loaders using a helper function
train_loader, val_loader = get_mnist_dataloaders(options["data_path"], batch_size)
# Initialize network
model = Net(dropout=dropout).to(device)
# Learning rate optimizer
optimizer = optim.Adam(model.parameters(), lr=lr)
scheduler = StepLR(optimizer, step_size=1, gamma=0.7)
# Network training & validation loop
for epoch in range(0, options["epochs"]):
avg_train_loss = train(
options, model, device, train_loader, optimizer, epoch
)
avg_val_loss = validate(model, device, val_loader)
if avg_val_loss <= best_val_loss:
best_val_loss = avg_val_loss
# Print intermediate validation & training loss
print(f"Epoch {epoch + 1} of {options['epochs']} --- average train loss: {avg_train_loss} --- average validation loss: {avg_val_loss}")
scheduler.step()
# Return the best validation loss
return best_val_loss
import model_use
import preprocessing
import training
print('Loading dataset')
data_texts, data_labels = datasets.load_test_data()
print('Loading model')
model, tokenizer = model_use.load_model()
print('Preprocessing on dataset')
data_inputs, _, _ = preprocessing.preprocess(data_texts, tokenizer=tokenizer)
# Remove neutral data and labels from the csv file, since this is a binary classification model
result = np.where(data_labels == 2)[0]
data_inputs = np.delete(data_inputs, result, axis=0)
data_labels = np.delete(data_labels, result, axis=0)
print('Running validation')
#K=2 in k fold cross validation. n_splits=2
results = training.validate(model,
data_inputs,
data_labels,
num_epoch=20,
n_splits=2)
print(results)
accuracies = [result[1][1] for result in results]
print('Fold Accuracies:', ', '.join([str(a) for a in accuracies]))
print('Average Accuracy:', sum(accuracies) / len(accuracies))
def train_and_validate(expected_numbers_of_worms_per_video, cluster_threshold=DEFAULT_CLUSTER_THRESHOLD, greyscale_threshold=DEFAULT_GREYSCALE_THRESHOLD):
cluster_threshold, greyscale_threshold = training.train_using_monte_carlo_simulation(expected_numbers_of_worms_per_video["training"])
return training.validate(expected_numbers_of_worms_per_video["validation"], cluster_threshold, greyscale_threshold)
softmax_proteins = next(iter(train_loader))[:4]
softmax_name = args.results_dir / Path("softmax.png")
if args.anneal_learning_rates and args.plot_learning_rates:
learning_rates = []
learning_rates_name = args.results_dir / Path("learning_rates.png")
try:
for epoch in range(1, args.epochs + 1):
start_time = time.time()
train_loss, train_metrics = train_epoch(
epoch, model, optimizer, train_loader, args.log_interval,
args.clip_grad_norm, args.clip_grad_value, scheduler)
if args.val_ratio > 0:
val_loss, val_metrics = validate(epoch, model, val_loader)
loss_str = "Validation"
loss_value_str = f"{val_loss:.5f}"
val_str = f"{loss_str} loss: {loss_value_str} "
improved = val_loss < best_loss
else:
loss_str = "Training"
loss_value_str = f"{train_loss:.5f}"
val_str = ""
improved = train_loss < best_loss
if improved:
# If model improved, save the model
model.save(model_save_name)
print(
def importance(model,
X_train,
y_train,
optimizer,
criterion,
epochs,
device,
val_dl,
prop,
writer=None,
early_stopping=True,
batch_size=64,
recompute_epoch=0):
# Compute initial importance values
unique_labels = len(list(set(y_train)))
importances = compute_samples_property(model,
X_train,
y_train,
prop=prop,
unique_labels=unique_labels,
indices=False,
verbose=True)
# Initialization
model.train()
curr_epoch, size = 0, len(importances)
rnd_indices = np.random.randint(0, len(X_train), batch_size)
running_loss, running_corrects, total = 0, 0, 0
train_losses, train_accuracies, val_losses, val_accuracies = [], [], [], []
if recompute_epoch == 0: recompute_epoch = epochs
# Simulate an epoch using len(X_train) = batch_size * batches
while curr_epoch < epochs:
if curr_epoch % recompute_epoch == 0 and curr_epoch != 0:
importances = compute_samples_property(model,
X_train,
y_train,
prop=prop,
unique_labels=unique_labels,
indices=False,
verbose=True)
seen_samples, running_loss = 0, 0
while seen_samples < len(X_train):
if curr_epoch == 0:
curr_indices = rnd_indices
else:
curr_indices = metropolis_hastings(importances, curr_indices)
inputs, labels = X_train[curr_indices], y_train[curr_indices]
inputs = np.moveaxis(inputs, source=-1,
destination=1).astype(np.float32)
inputs, labels = torch.Tensor(inputs), torch.LongTensor(labels)
inputs, labels = inputs.to(device), labels.to(device)
optimizer.zero_grad()
outputs = model(inputs)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
_, predicted = torch.max(outputs.data, 1)
labels = labels.long()
running_corrects += (predicted == labels).sum().item()
running_loss += loss
total += labels.size(0)
seen_samples += len(curr_indices)
# Train loss/accuracy
train_loss = running_loss
train_acc = running_corrects / total
train_losses.append(train_loss)
train_accuracies.append(train_acc)
# Validation loss/accuracy
val_loss, val_acc = validate(model,
val_dl,
criterion,
device,
flatten=False)
val_losses.append(val_loss)
val_accuracies.append(val_acc)
curr_epoch += 1
print(
'[ EPOCH {}: train_loss={} | train_acc={} | val_loss={} | val_acc={} ]'
.format(curr_epoch, train_loss, train_acc, val_loss, val_acc))
opponent_value_function = None
opponent_agent = BareAgent(opponent_policy, opponent_value_function)
opponent_agents = train(player_agent_train, opponent_agent, 2001, 1000, 500)
# player_value_function.save('SARSA_MAX_backward__UCB_vs_Self')
print("ended training!")
player_policy_val = GreedyPolicy()
player_agent_val = BareAgent(player_policy_val, player_value_function)
# opponent_policy = RandomPolicy()
# opponent_value_function = None
# opponent_agent = BareAgent(opponent_policy, opponent_value_function)
# opponent_agents = [opponent_agent]
validate(player_agent_val, opponent_agents, 501)
# ## ------------------------------- ####
#
#
# print("ended validation")
## ------------------------------- ####
# # opponent_policy = RandomPolicy()
# opponent_policy = GreedyPolicy()
# opponent_value_function = LookupTable()
# opponent_value_function.load('SARSA_MAX_backward__UCB_vs_Self')
# opponent_agent = BareAgent(opponent_policy, opponent_value_function)
#
# while True:
# play_human(opponent_agent, True)
train_loss_count += 1
print_seqs_count += batch_size
if print_seqs_count >= print_every_samples:
process = psutil.Process(os.getpid())
print(f'Progress: {100 * seqs_processed / train_seqs_per_epoch:6.3f}% of epoch. Total time: {readable_time(time.time() - print_seqs_overall_time):>7s}. Iteration time: {readable_time(time.time() - print_seqs_iteration_time):>7s} CPU Memory: {process.memory_info().rss / (1024**3)} GiB')
print_seqs_iteration_time = time.time()
print_seqs_count = 0
if seqs_processed >= train_seqs_per_epoch:
epoch += 1
train_loss = acc_train_loss / train_loss_count
seqs_processed = 0
acc_train_loss = 0
train_loss_count = 0
val_loss, _ = validate(epoch, model, val_loader)
# If save the latest model
if args.multi_gpu:
model.module.save(model_save_name_latest)
else:
model.save(model_save_name_latest)
improved = val_loss < best_val_loss
if improved:
# If model improved, save the model
if args.multi_gpu:
model.module.save(model_save_name)
else:
model.save(model_save_name)
def compare_approaches(save_fig=True):
# Setup
date = datetime.now().strftime("%m-%d_%H-%M-%S")
toy = ToyExperiment(n_features=2,
hidden_dim=5,
n_classes=2,
n_samples=500,
theta=90,
N_EPOCHS=20,
convert_y=False)
ss = StandardScaler()
X_train_sc = ss.fit_transform(toy.X_train)
X_train_noisy_sc = ss.fit_transform(toy.X_train_noisy)
X_valid_noisy_sc = ss.fit_transform(toy.X_valid_noisy)
# Compute loss and accuracy of clean model on noisy dataset
train_dl = get_data_loader(X_train_noisy_sc, toy.y_train)
valid_dl = get_data_loader(X_valid_noisy_sc, toy.y_valid)
train_loss_clean, train_acc_clean = validate(toy.model_clean,
train_dl,
toy.criterion,
device,
flatten=True)
valid_loss_clean, valid_acc_clean = validate(toy.model_clean,
valid_dl,
toy.criterion,
device,
flatten=True)
# Percentage to keep
perc = 0.5
n_idx = int(len(X_train_sc) * perc)
# Leave one out
loo_losses = get_leave_one_out_losses(toy, dset='valid')
loo_idx = np.argsort(loo_losses)
loo_idx = loo_idx[::-1][:n_idx]
# leave_one_out_idx = np.random.choice(len(X_train_noisy_sc), n_idx, replace=False)
# Entropy
entropy_idx = compute_samples_property(toy.model_clean,
toy.X_train_noisy,
toy.y_train,
prop='entropy',
flatten=True,
indices=True,
unique_labels=[0, 1])
entropy_idx = entropy_idx[::-1][:n_idx]
# Hessian
hessian_losses = instance_selection(model=toy.model_clean,
X_train=X_train_noisy_sc,
y_train=toy.y_train,
sparse=False,
X_valid=X_valid_noisy_sc,
y_valid=toy.y_valid,
criterion=toy.criterion,
treshold=False,
return_influences=True)
hessian_idx = np.argsort(hessian_losses)[:n_idx]
# Jacobian
jacobian_losses = instance_selection_no_hessian(model=toy.model_clean,
X_train=X_train_noisy_sc,
y_train=toy.y_train,
X_valid=X_valid_noisy_sc,
y_valid=toy.y_valid,
criterion=toy.criterion,
flatten=True,
return_influences=True)
jacobian_idx = np.argsort(jacobian_losses)[:n_idx]
# Training derivatives
train_derivatives = instance_selection_train_derivatives(
model=toy.model_clean,
X=X_train_noisy_sc,
y=toy.y_train,
criterion=toy.criterion,
flatten=True)
train_derivatives_idx = np.argsort(train_derivatives)[:n_idx]
# n_idx = min(len(hessian_idx), len(jacobian_idx))
# Random
random_idx = np.random.choice(len(X_train_noisy_sc), n_idx, replace=False)
# Generate scatterplot
methods = [(random_idx, 'random'), (loo_idx, 'leave one out'),
(entropy_idx, 'entropy'), (hessian_idx, 'hessian'),
(jacobian_idx, 'jacobian'),
[train_derivatives_idx, 'train derivatives norms']]
create_scatterplot(X_train_sc,
X_train_noisy_sc,
toy.y_train,
X_valid_noisy_sc,
toy.y_valid,
methods,
n_idx,
date=date)
# Print common selected indices between loo and jacobian
common = len(list(set(loo_idx) & set(jacobian_idx)))
print('Common indices LOO and Jacobian: {}/{} | {:.2f}%'.format(
common, n_idx, common / n_idx * 100))
# Train
plt.figure(figsize=(15, 5))
for idx, name in tqdm(methods, desc='Training'):
model_ = copy.deepcopy(toy.model_clean)
X_train_ = X_train_noisy_sc[idx][:n_idx]
y_train_ = toy.y_train[idx][:n_idx]
optimizer = torch.optim.SGD(model_.parameters(), lr=0.01)
train_dl_ = get_data_loader(X_train_, y_train_)
train_losses_, train_accs_, val_accs_, val_losses_, epoch_, _ = train(
model=model_,
train_loader=train_dl_,
val_loader=toy.valid_noisy_dl,
test_loader=toy.test_noisy_dl,
optimizer=optimizer,
criterion=toy.criterion,
device=device,
epochs=N_EPOCHS,
early_stopping=False,
flatten=True,
verbose=False)
# Make accuracies and losses starting from the same intial point
train_losses_.insert(0, train_loss_clean)
train_accs_.insert(0, train_acc_clean)
val_losses_.insert(0, valid_loss_clean)
val_accs_.insert(0, valid_acc_clean)
# Plot
plt.subplot(1, 2, 1)
plt.plot(range(N_EPOCHS + 1), train_accs_, label=name)
plt.xlabel('Epochs', fontsize=16)
plt.ylabel('Accuracy', fontsize=16)
plt.title('Train\n')
plt.legend(loc='best')
plt.subplot(1, 2, 2)
plt.plot(range(N_EPOCHS + 1), val_accs_, label=name)
plt.xlabel('Epochs', fontsize=16)
plt.ylabel('Accuracy', fontsize=16)
plt.title('Validation\n')
plt.legend(loc='best')
if save_fig:
plt.savefig(ROOT_DIR +
'/toy-model/images/compare_{}_accuracy.png'.format(date))
plt.show()
def run(args):
print('\nSettings: \n', args, '\n')
args.model_signature = str(dt.datetime.now())[0:19].replace(' ', '_')
args.model_signature = args.model_signature.replace(':', '_')
########## Find GPUs
(gpu_config, n_gpu_used) = set_gpus(args.n_gpu)
########## Data, model, and optimizer setup
mnist = MNIST(args)
x = tf.placeholder(tf.float32, [None, 28, 28, 1])
if args.model == 'hvae':
if not args.K:
raise ValueError('Must set number of flow steps when using HVAE')
elif not args.temp_method:
raise ValueError('Must set tempering method when using HVAE')
model = HVAE(args, mnist.avg_logit)
elif args.model == 'cnn':
model = VAE(args, mnist.avg_logit)
else:
raise ValueError('Invalid model choice')
elbo = model.get_elbo(x, args)
nll = model.get_nll(x, args)
optimizer = AdamaxOptimizer(learning_rate=args.learn_rate,
eps=args.adamax_eps)
opt_step = optimizer.minimize(-elbo)
########## Tensorflow and saver setup
sess = tf.Session(config=gpu_config)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
savepath = os.path.join(args.checkpoint_dir, args.model_signature,
'model.ckpt')
if not os.path.exists(args.checkpoint_dir):
os.makedirs(args.checkpoint_dir)
########## Test that GPU memory is sufficient
if n_gpu_used > 0:
try:
x_test = mnist.next_test_batch()
(t_e, t_n) = sess.run((elbo, nll), {x: x_test})
mnist.batch_idx_test = 0 # Reset batch counter if it works
except:
raise MemoryError("""
Likely insufficient GPU memory
Reduce test batch by lowering the -tbs parameter
""")
########## Training Loop
train_elbo_hist = []
val_elbo_hist = []
# For early stopping
best_elbo = -np.inf
es_epochs = 0
epoch = 0
train_times = []
for epoch in range(1, args.epochs + 1):
t0 = time.time()
train_elbo = train(epoch, mnist, opt_step, elbo, x, args, sess)
train_elbo_hist.append(train_elbo)
train_times.append(time.time() - t0)
print('One epoch took {:.2f} seconds'.format(time.time() - t0))
val_elbo = validate(mnist, elbo, x, sess)
val_elbo_hist.append(val_elbo)
if val_elbo > best_elbo:
# Save the model that currently generalizes best
es_epochs = 0
best_elbo = val_elbo
saver.save(sess, savepath)
best_model_epoch = epoch
elif args.early_stopping_epochs > 0:
es_epochs += 1
if es_epochs >= args.early_stopping_epochs:
print('***** STOPPING EARLY ON EPOCH {} of {} *****'.format(
epoch, args.epochs))
break
print('--> Early stopping: {}/{} (Best ELBO: {:.4f})'.format(
es_epochs, args.early_stopping_epochs, best_elbo))
print('\t Current val ELBO: {:.4f}\n'.format(val_elbo))
if np.isnan(val_elbo):
raise ValueError('NaN encountered!')
train_times = np.array(train_times)
mean_time = np.mean(train_times)
std_time = np.std(train_times)
print('Average train time per epoch: {:.2f} +/- {:.2f}'.format(
mean_time, std_time))
########## Evaluation
# Restore the best-performing model
saver.restore(sess, savepath)
test_elbos = np.zeros(args.n_nll_runs)
test_nlls = np.zeros(args.n_nll_runs)
for i in range(args.n_nll_runs):
print('\n---- Test run {} of {} ----\n'.format(i + 1, args.n_nll_runs))
(test_elbos[i], test_nlls[i]) = evaluate(mnist, elbo, nll, x, args,
sess)
mean_elbo = np.mean(test_elbos)
std_elbo = np.std(test_elbos)
mean_nll = np.mean(test_nlls)
std_nll = np.std(test_nlls)
print('\nTest ELBO: {:.2f} +/- {:.2f}'.format(mean_elbo, std_elbo))
print('Test NLL: {:.2f} +/- {:.2f}'.format(mean_nll, std_nll))
########## Logging, Saving, and Plotting
with open(args.logfile, 'a') as ff:
print('----------------- Test ID {} -----------------'.format(
args.model_signature),
file=ff)
print(args, file=ff)
print('Stopped after {} epochs'.format(epoch), file=ff)
print('Best model from epoch {}'.format(best_model_epoch), file=ff)
print('Average train time per epoch: {:.2f} +/- {:.2f}'.format(
mean_time, std_time),
file=ff)
print('FINAL VALIDATION ELBO: {:.2f}'.format(val_elbo_hist[-1]),
file=ff)
print('Test ELBO: {:.2f} +/- {:.2f}'.format(mean_elbo, std_elbo),
file=ff)
print('Test NLL: {:.2f} +/- {:.2f}\n'.format(mean_nll, std_nll),
file=ff)
if not os.path.exists(args.pickle_dir):
os.makedirs(args.pickle_dir)
train_dict = {
'train_elbo': train_elbo_hist,
'val_elbo': val_elbo_hist,
'args': args
}
pickle.dump(
train_dict,
open(os.path.join(args.pickle_dir, args.model_signature + '.p'), 'wb'))
if not os.path.exists(args.plot_dir):
os.makedirs(args.plot_dir)
tf_gen_samples = model.get_samples(args)
np_gen_samples = sess.run(tf_gen_samples)
plot_digit_samples(np_gen_samples, args)
plot_training_curve(train_elbo_hist, val_elbo_hist, args)
########## Email notification upon test completion
try:
msg_text = """Test completed for ID {0}.
Parameters: {1}
Test ELBO: {2:.2f} +/- {3:.2f}
Test NLL: {4:.2f} +/- {5:.2f} """.format(args.model_signature, args,
mean_elbo, std_elbo, mean_nll,
std_nll)
msg = MIMEText(msg_text)
msg['Subject'] = 'Test ID {0} Complete'.format(args.model_signature)
msg['To'] = args.receiver
msg['From'] = args.sender
s = smtplib.SMTP('localhost')
s.sendmail(args.sender, [args.receiver], msg.as_string())
s.quit()
except:
print('Unable to send email from sender {0} to receiver {1}'.format(
args.sender, args.receiver))