def plot_summary_graphs_layers(vals_per_epoch, n_type, v_type, im_path):
vals_np = np.array(vals_per_epoch)
for it in range(vals_np.shape[1]):
val = vals_np[:, it].tolist()
if it % 2:
display_results.plot_graph(val, None, "{}_{}_layer_bias_{}".format(n_type, v_type, it), im_path)
else:
display_results.plot_graph(val, None, "{}_{}_layer_weight_{}".format(n_type, v_type, it), im_path)
return
def plot_summary_graphs_layers(vals_to_plot, v_type, im_path):
vals_np = np.array(vals_to_plot)
for it in range(vals_np.shape[1]):
val = vals_np[:, it].tolist()
layer_indx = it // 2
if it % 2: # odd row numbers store bias vals
display_results.plot_graph(
val, None, "{}_layer_bias_{}".format(v_type, layer_indx),
im_path)
print('{}_layer_bias_{} graph plotted'.format(v_type, layer_indx))
else: # even row numbers store weight vals
display_results.plot_graph(
val, None, "{}_layer_weight_{}".format(v_type, layer_indx),
im_path)
print('{}_layer_weight_{} graph plotted'.format(
v_type, layer_indx))
return
betas=(params.beta1, params.beta2))
# d_optimizer = optim.SGD(discriminator.parameters(), lr=params.learning_rate)
# g_optimizer = optim.SGD(generator.parameters(), lr=params.learning_rate)
# fetch loss function
loss_fn = gan_net.real_fake_loss_fn
loss_fn = gan_net.disc_dist_loss_func
loss_fn = gan_net.cont_dist_loss_func
# Train the model
logging.info("Starting training for {} epoch(s)".format(params.num_epochs))
num_test_samples = 20
test_noise = gan_net.noise(num_test_samples, params.noise_dim)
D_losses = []
G_losses = []
D_preds = []
G_preds = []
train_gan(discriminator, generator, train_dl, dev_dl, d_optimizer,
g_optimizer, loss_fn, params, args.model_dir, args.restore_file)
# track results
display_results.plot_graph(G_losses, D_losses, "Loss")
display_results.plot_graph(G_preds, D_preds, "Predictions")
d_grads_graph = collect_network_statistics(discriminator)
g_grads_graph = collect_network_statistics(generator)
display_results.plot_graph(g_grads_graph, d_grads_graph, "Grads")
# comment to self - print to file the name of loss function !!!!!!
# print(loss_fn)
metrics = net.metrics
incorrect = net.incorrect
correct_fn = net.correct_classification
losses = []
accuracy = []
grads_per_epoch = []
vals_per_epoch = []
# Train the model
logging.info("Starting training for {} epoch(s)".format(params.num_epochs))
train_and_evaluate(model, train_dl, dev_dl, optimizer, loss_fn, metrics,
incorrect, correct_fn, params, args.model_dir,
args.restore_file)
print('plotting graphs')
display_results.plot_graph(losses, None, "General Loss", args.model_dir)
print('loss graph plotted')
display_results.plot_graph(accuracy, None, "General dev accuracy",
args.model_dir)
print('accuracy graph plotted')
plot_summary_graphs_layers(grads_per_epoch, 'Grads', args.model_dir)
# grads_np = np.array(grads_per_epoch)
# for i in range(grads_np.shape[1]):
# val = grads_np[:, i].tolist()
# display_results.plot_graph(val, None, "Grads_layer_{}".format(i+1), args.model_dir)
test_label_after, params.num_classes).to(device)
test_one_hot_v = torch.cat((test_bef_one_hot_v, test_aft_one_hot_v), 1)
losses_dict = {'D_losses': [], 'G_losses': []}
preds_dict = {'D_preds': [], 'G_preds': []}
accuracies_dict = {'real_accuracy_vals': [], 'fake_accuracy_vals': []}
grads_dict = {'grads_per_epoch_d': [], 'grads_per_epoch_g': []}
vals_dict = {'vals_per_epoch_d': [], 'vals_per_epoch_g': []}
train_gan(discriminator, generator, train_dl, d_optimizer, g_optimizer,
real_fake_loss_fn, class_selection_loss_fn, params,
args.model_dir)
# track results
print('plotting graphs')
display_results.plot_graph(losses_dict['G_losses'],
losses_dict['D_losses'], "Loss", args.model_dir)
print('loss graph plotted')
display_results.plot_graph(preds_dict['G_preds'], preds_dict['D_preds'],
"Predictions", args.model_dir)
print('predictions graph plotted')
display_results.plot_graph(accuracies_dict['real_accuracy_vals'],
accuracies_dict['fake_accuracy_vals'],
"Accuracy", args.model_dir)
print('accuracy graph plotted')
plot_summary_graphs_layers(grads_dict['grads_per_epoch_g'], 'G', 'Grads',
args.model_dir)
plot_summary_graphs_layers(grads_dict['grads_per_epoch_d'], 'D', 'Grads',
args.model_dir)
plot_summary_graphs_layers(vals_dict['vals_per_epoch_g'], 'G', 'Vals',
args.model_dir)
def train_g(g_model, train_dataloader, g_optimizer, mse_loss_fn, params, model_dir):
best_loss = np.inf
dest_min = 0
dest_max = 255
curr_min = -1
curr_max = 1
fig = display_results.create_figure()
for epoch in range(params.num_epochs):
# Run one epoch
logging.info("Epoch {}/{}".format(epoch + 1, params.num_epochs))
test_samples, real_samples, loss_mean = train(g_model, g_optimizer, mse_loss_fn, train_dataloader, params, epoch, fig)
is_best = loss_mean <= best_loss
if is_best:
logging.info("- Found new best loss")
print("Epoch {}/{}".format(epoch + 1, params.num_epochs))
print("- Found new best loss")
best_loss = loss_mean
print("mean loss is {:05.3f}".format(loss_mean))
loss_metric_dict = {'loss': loss_mean}
utils.save_checkpoint({'epoch': epoch + 1,
'state_dict': g_model.state_dict(),
'optim_dict': g_optimizer.state_dict()}, is_best=is_best, checkpoint=model_dir)
# Save best val metrics in a json file in the model directory
best_json_path = os.path.join(model_dir, "metrics_min_avg_loss_best_weights.json")
utils.save_dict_to_json(loss_metric_dict, best_json_path)
best_g_grads_graph = collect_network_statistics(g_model)
display_results.plot_graph(best_g_grads_graph, [], "Grads_Best", args.model_dir, epoch=epoch+1)
if test_samples is not None:
np_test_samples = np.array(test_samples)
np_test_samples = \
dest_min + (dest_max - dest_min) * (np_test_samples - curr_min) / (curr_max - curr_min)
np_test_samples = np.around(np_test_samples).astype(int)
np_test_out = (test_noise.cpu().numpy())
np_test_labels = (test_labels.view(test_labels.shape[0], -1).cpu().numpy())
data_path = os.path.join(model_dir, 'data')
if not os.path.isdir(data_path):
os.mkdir(data_path)
test_all_data = (np.concatenate((np_test_samples, np_test_out, np_test_labels), axis=1)).tolist()
last_csv_path = os.path.join(data_path, "best_samples_epoch_{}.csv".format(epoch + 1))
utils.save_incorrect_to_csv(test_all_data, last_csv_path)
if test_samples is not None:
utils.save_checkpoint({'epoch': epoch + 1,
'state_dict': g_model.state_dict(),
'optim_dict': g_optimizer.state_dict()}, is_best=False, checkpoint=model_dir,
ntype='g')
np_test_samples = np.array(test_samples)
np_test_samples = \
dest_min + (dest_max - dest_min) * (np_test_samples - curr_min) / (curr_max - curr_min)
np_test_samples = np.around(np_test_samples).astype(int)
np_test_out = (test_noise.cpu().numpy())
np_test_labels = (test_labels.view(test_labels.shape[0], -1).cpu().numpy())
data_path = os.path.join(model_dir, 'data')
if not os.path.isdir(data_path):
os.mkdir(data_path)
test_all_data = (np.concatenate((np_test_samples, np_test_out, np_test_labels), axis=1)).tolist()
last_csv_path = os.path.join(data_path, "samples_epoch_{}.csv".format(epoch+1))
utils.save_incorrect_to_csv(test_all_data, last_csv_path)
display_results.close_figure(fig)
return
logging.info("Starting training for {} epoch(s)".format(params.num_epochs))
num_test_samples = 20
test_noise = gan_net.noise(num_test_samples, params.noise_dim)
# test_labels = list(range(num_test_samples))
# test_labels = [it % params.num_classes for it in test_labels]
test_labels = [0 for _ in range(num_test_samples)]
# test_labels = [9 for _ in range(num_test_samples)] # temp
# test_labels = [8 for _ in range(num_test_samples)] # temp
test_labels = torch.Tensor(test_labels)
test_labels = test_labels.type(torch.LongTensor)
test_labels = test_labels.to(device)
test_labels = test_labels.view(num_test_samples, -1)
test_one_hot_v = gan_net.convert_int_to_one_hot_vector(test_labels, params.num_classes).to(device)
D_losses = []
G_losses = []
D_preds = []
G_preds = []
accuracy_vals = []
train_g(generator, train_dl, g_optimizer, mse_loss_fn, params, args.model_dir)
# track results
display_results.plot_graph(G_losses, D_losses, "Loss", args.model_dir)
# display_results.plot_graph(G_preds, D_preds, "Predictions", args.model_dir)
last_g_grads_graph = collect_network_statistics(generator)
display_results.plot_graph(last_g_grads_graph, [], "Grads", args.model_dir)
# # test_labels = [0 for _ in range(num_test_samples)]
# test_labels = torch.Tensor(test_labels)
# test_labels = test_labels.type(torch.LongTensor)
# test_labels = test_labels.to(device)
#
# test_labels = test_labels.view(num_test_samples, -1)
# test_one_hot_v = gan_net.convert_int_to_one_hot_vector(test_labels, params.num_classes).to(device)
D_losses = []
G_losses = []
D_preds = []
G_preds = []
grads_per_epoch_g = []
grads_per_epoch_d = []
vals_per_epoch_g = []
vals_per_epoch_d = []
train_gan(discriminator, generator, train_dl, dev_dl, d_optimizer, g_optimizer, loss_fn, params, args.model_dir,
args.restore_file)
# track results
display_results.plot_graph(G_losses, D_losses, "Loss", args.model_dir)
display_results.plot_graph(G_preds, D_preds, "Predictions", args.model_dir)
plot_summary_graphs_layers(grads_per_epoch_g, 'G', 'Grads', args.model_dir)
plot_summary_graphs_layers(grads_per_epoch_d, 'D', 'Grads', args.model_dir)
plot_summary_graphs_layers(vals_per_epoch_g, 'G', 'Vals', args.model_dir)
plot_summary_graphs_layers(vals_per_epoch_d, 'D', 'Vals', args.model_dir)
logging.info("network structure is")
logging.info("{}".format(model))
optimizer = optim.Adam(model.parameters(),
lr=params.learning_rate,
betas=(params.beta1, params.beta2))
# fetch loss function and metrics
# loss_fn = vae_net.uniform_loss_fn
loss_fn = vae_net.loss_fn
# metrics = net.metrics
# incorrect = net.incorrect
losses = []
# uni_losses = []
bce_losses = []
kl_losses = []
# Train the model
logging.info("Starting training for {} epoch(s)".format(params.num_epochs))
train_vae(model, train_dl, optimizer, loss_fn, params, args.model_dir)
display_results.plot_graph(losses, None, "General Loss", args.model_dir)
# display_results.plot_graph(uni_losses, kl_losses, "VAE Loss", args.model_dir)
display_results.plot_graph(bce_losses, kl_losses, "VAE Loss",
args.model_dir)
grads_graph = collect_network_statistics(model)
display_results.plot_graph(grads_graph, None, "Grads", args.model_dir)
