iter_str: total_iters,
train_nll_str: numpy.mean(costs),
valid_nll_str: valid_cost,
test_nll_str: test_cost,
lowest_valid_str: lowest_valid_cost,
corresp_test_str: corresponding_test_cost,
'train time': total_time,
'valid time': valid_time,
'test time': test_time,
'wall clock time': time() - exp_start
}
lib.save_training_info(training_info, FOLDER_PREFIX)
print "Train info saved!",
y_axis_strs = [train_nll_str, valid_nll_str, test_nll_str]
lib.plot_traing_info(iter_str, y_axis_strs, FOLDER_PREFIX)
print "And plotted!"
# 5. Generate and save samples (time consuming)
# If not successful, we still have the params to sample afterward
print "Sampling!",
# Generate samples
generate_and_save_samples(tag)
print "Done!"
if total_iters-last_print_iters == PRINT_ITERS \
or total_time-last_print_time >= PRINT_TIME:
# If we are here b/c of onom_end_of_batch, we shouldn't mess
# with costs and last_print_iters
costs = []
last_print_time += PRINT_TIME
training_info = {'epoch' : epoch,
x_axis_str : total_iters,
train_nll_str : numpy.mean(costs),
valid_nll_str : valid_cost,
test_nll_str : test_cost,
'lowest valid cost' : lowest_valid_cost,
'correponding test cost' : corresponding_test_cost,
'train time' : total_time,
'valid time' : valid_time,
'test time' : test_time,
'wall clock time' : time.time()-exp_start}
lib.save_training_info(training_info, FOLDER_PREFIX)
print "Train info saved!",
y_axis_strs = [train_nll_str, valid_nll_str, test_nll_str]
lib.plot_traing_info(x_axis_str, y_axis_strs, FOLDER_PREFIX)
print "Plotted!"
# Generate and save samples
print "Sampling!",
tag = "e{}_i{}_t{:.2f}_tr{:.4f}_v{:.4f}"
tag = tag.format(epoch,
total_iters,
total_time/3600,
numpy.mean(cost),
valid_cost)
tag += ("_best" if new_lowest_cost else "")
# Generate samples
generate_and_save_samples(tag)
print "Done!"
