# Save and graph training progress x_axis_str = 'iter' train_nll_str, valid_nll_str, test_nll_str = \ 'train NLL (bits)', 'valid NLL (bits)', 'test NLL (bits)' training_info = {'epoch' : epoch, x_axis_str : total_iters, train_nll_str : numpy.mean(costs), valid_nll_str : valid_cost, test_nll_str : test_cost, 'lowest valid cost' : lowest_valid_cost, 'correponding test cost' : corresponding_test_cost, 'train time' : total_time, 'valid time' : valid_time, 'test time' : test_time, 'wall clock time' : time.time()-exp_start} lib.save_training_info(training_info, FOLDER_PREFIX) print "Train info saved!", y_axis_strs = [train_nll_str, valid_nll_str, test_nll_str] lib.plot_traing_info(x_axis_str, y_axis_strs, FOLDER_PREFIX) print "Plotted!" # Generate and save samples print "Sampling!", tag = "e{}_i{}_t{:.2f}_tr{:.4f}_v{:.4f}" tag = tag.format(epoch, total_iters, total_time/3600, numpy.mean(cost), valid_cost) tag += ("_best" if new_lowest_cost else "")
# 4. Save and graph training progress (fast) training_info = { epoch_str: epoch, iter_str: total_iters, train_nll_str: numpy.mean(costs), valid_nll_str: valid_cost, test_nll_str: test_cost, lowest_valid_str: lowest_valid_cost, corresp_test_str: corresponding_test_cost, 'train time': total_time, 'valid time': valid_time, 'test time': test_time, 'wall clock time': time() - exp_start } lib.save_training_info(training_info, FOLDER_PREFIX) print "Train info saved!", y_axis_strs = [train_nll_str, valid_nll_str, test_nll_str] lib.plot_traing_info(iter_str, y_axis_strs, FOLDER_PREFIX) print "And plotted!" #""" # 5. Generate and save samples (time consuming) # If not successful, we still have the params to sample afterward print "Sampling!", # Generate samples generate_and_save_samples(tag) print "Done!" if total_iters-last_print_iters == PRINT_ITERS \ or total_time-last_print_time >= PRINT_TIME: