def start_all_logging_instruments(hyper, results_path, test_images): writer = tf.summary.create_file_writer(logdir=results_path) logger = setup_logger(log_file_name=append_timestamp_to_file(file_name=results_path + '/loss.log', termination='.log'), logger_name=append_timestamp_to_file('logger', termination='')) log_all_hyperparameters(hyper=hyper, logger=logger) plot_originals(test_images=test_images, results_path=results_path) return writer, logger
def start_all_logging_instruments(hyper, test_images): results_path = determine_path_to_save_results( model_type=hyper['model_type'], dataset_name=hyper['dataset_name']) if not os.path.exists(results_path): os.mkdir(results_path) logger = setup_logger(log_file_name=append_timestamp_to_file( file_name=results_path + '/loss.log', termination='.log'), logger_name=append_timestamp_to_file('logger', termination='')) log_all_hyperparameters(hyper=hyper, logger=logger) plot_originals(test_images=test_images, results_path=results_path) return logger, results_path
def train_vae_model(vae_opt, model, writer, hyper, train_dataset, test_dataset, logger, results_path, test_images, monitor_gradients=False): (iteration_counter, results_file, hyper_file, cont_c_linspace, disc_c_linspace) = initialize_vae_variables(results_path=results_path, hyper=hyper) grad_monitor_dict = {} grad_norm = tf.constant(0., dtype=tf.float32) with open(file=hyper_file, mode='wb') as f: pickle.dump(obj=hyper, file=f) with writer.as_default(): initial_time = time.time() for epoch in range(1, hyper['epochs'] + 1): t0 = time.time() train_loss_mean = tf.keras.metrics.Mean() for x_train in train_dataset.take(hyper['iter_per_epoch']): output = vae_opt.compute_gradients(x=x_train) gradients, loss, log_px_z, kl, kl_n, kl_d = output vae_opt.apply_gradients(gradients=gradients) iteration_counter += 1 train_loss_mean(loss) append_train_summaries(tracking_losses=output, iteration_counter=iteration_counter) update_regularization_channels(vae_opt=vae_opt, iteration_counter=iteration_counter, disc_c_linspace=disc_c_linspace, cont_c_linspace=cont_c_linspace) if monitor_gradients: grad_norm = vae_opt.monitor_parameter_gradients_at_psi(x=x_train) grad_monitor_dict.update({iteration_counter: grad_norm.numpy()}) with open(file='./Results/gradients_' + str(epoch) + '.pkl', mode='wb') as f: pickle.dump(obj=grad_monitor_dict, file=f) t1 = time.time() evaluate_progress_in_test_set(epoch=epoch, test_dataset=test_dataset, vae_opt=vae_opt, hyper=hyper, logger=logger, iteration_counter=iteration_counter, train_loss_mean=train_loss_mean, time_taken=t1 - t0, grad_norm=grad_norm) if epoch % 10 == 0: model.save_weights(filepath=append_timestamp_to_file(results_file, '.h5')) plot_reconstructions_samples_and_traversals(model=model, hyper=hyper, epoch=epoch, results_path=results_path, test_images=test_images, vae_opt=vae_opt) writer.flush() final_time = time.time() logger.info(f'Total training time {final_time - initial_time: 4.1f} secs') logger.info(f'Final temp {vae_opt.temp.numpy(): 4.5f}') results_file = append_timestamp_to_file(file_name=results_file, termination='.h5') model.save_weights(filepath=results_file)
def train_sop(sop_optimizer, hyper, train_dataset, test_dataset, logger): initial_time = time.time() iteration_counter = 0 for epoch in range(1, hyper['epochs'] + 1): train_mean_loss = tf.keras.metrics.Mean() tic = time.time() for x_train in train_dataset.take(hyper['iter_per_epoch']): x_train_lower = x_train[:, 14:, :, :] x_train_upper = x_train[:, :14, :, :] gradients, loss = sop_optimizer.compute_gradients_and_loss( x_upper=x_train_upper, x_lower=x_train_lower) sop_optimizer.apply_gradients(gradients=gradients) train_mean_loss(loss) iteration_counter += 1 toc = time.time() evaluate_progress_in_test_set(epoch=epoch, sop_optimizer=sop_optimizer, test_dataset=test_dataset, logger=logger, hyper=hyper, iteration_counter=iteration_counter, time_taken=toc - tic, train_mean_loss=train_mean_loss) final_time = time.time() logger.info(f'Total training time {final_time - initial_time: 4.1f} secs') results_file = f'./Log/model_weights_{sop_optimizer.model.model_type}.h5' results_file = append_timestamp_to_file(file_name=results_file, termination='.h5') sop_optimizer.model.save_weights(filepath=results_file)
def save_final_results(nets, logger, results_file, initial_time, temp): final_time = time.time() logger.info(f'Total training time {final_time - initial_time: 4.1f} secs') logger.info(f'Final temp {temp: 4.5f}') results_file = append_timestamp_to_file(file_name=results_file, termination='.h5') nets.save_weights(filepath=results_file)
def run_sop(hyper, results_path, data): train_dataset, test_dataset = data sop_optimizer = setup_sop_optimizer(hyper=hyper) logger = setup_logger(log_file_name=append_timestamp_to_file( file_name=results_path + f'/loss_{sop_optimizer.model.model_type}.log', termination='.log')) log_all_hyperparameters(hyper=hyper, logger=logger) train_sop(sop_optimizer=sop_optimizer, hyper=hyper, train_dataset=train_dataset, test_dataset=test_dataset, logger=logger)
def train_sop(sop_optimizer, hyper, train_dataset, test_dataset, logger): initial_time = time.time() iteration_counter = 0 sample_size = hyper['sample_size'] for epoch in range(1, hyper['epochs'] + 1): train_mean_loss = tf.keras.metrics.Mean() tic = time.time() for x_train in train_dataset.take(hyper['iter_per_epoch']): x_train_lower = x_train[:, 14:, :, :] x_train_upper = x_train[:, :14, :, :] gradients, loss = sop_optimizer.compute_gradients_and_loss( x_upper=x_train_upper, x_lower=x_train_lower, sample_size=sample_size) sop_optimizer.apply_gradients(gradients=gradients) update_learning_rate(sop_optimizer, epoch, iteration_counter, hyper) train_mean_loss(loss) iteration_counter += 1 time_taken = time.time() - tic if epoch % hyper['check_every'] == 0 or epoch == hyper['epochs']: evaluate_progress(epoch=epoch, sop_optimizer=sop_optimizer, test_dataset=test_dataset, train_dataset=train_dataset, logger=logger, hyper=hyper, train_mean_loss=train_mean_loss, iteration_counter=iteration_counter, tic=tic) else: logger.info( f'Epoch {epoch:4d} || Test_Recon 0.00000e+00 || ' f'Train_Recon {train_mean_loss.result().numpy():2.3e} || ' f'Temp {sop_optimizer.model.temp:2.1e} || ' f'{sop_optimizer.model.model_type} || ' f'{sop_optimizer.optimizer.learning_rate.numpy():1.1e} || ' f'Time {time_taken:4.1f} sec') final_time = time.time() logger.info(f'Total training time {final_time - initial_time: 4.1f} secs') results_file = f'./Log/model_weights_{sop_optimizer.model.model_type}.h5' results_file = append_timestamp_to_file(file_name=results_file, termination='.h5') sop_optimizer.model.save_weights(filepath=results_file)
def run_sop(hyper, results_path): tf.random.set_seed(seed=hyper['seed']) data = load_mnist_sop_data(batch_n=hyper['batch_size']) train_dataset, test_dataset = data sop_optimizer = setup_sop_optimizer(hyper=hyper) model_type = sop_optimizer.model.model_type log_path = results_path + f'/loss_{model_type}.log' logger = setup_logger(log_file_name=append_timestamp_to_file( file_name=log_path, termination='.log'), logger_name=model_type + str(hyper['seed'])) log_all_hyperparameters(hyper=hyper, logger=logger) save_hyper(hyper) train_sop(sop_optimizer=sop_optimizer, hyper=hyper, train_dataset=train_dataset, test_dataset=test_dataset, logger=logger)
def determine_path_to_save_results(model_type, dataset_name): results_path = './Log/' + dataset_name + '_' + model_type + append_timestamp_to_file( '', termination='') return results_path
def save_intermediate_results(epoch, vae_opt, test_images, hyper, results_file, results_path): if epoch % hyper['save_every'] == 0: vae_opt.nets.save_weights( filepath=append_timestamp_to_file(results_file, '.h5'))