def train(self): """ Performs the training process given the configuration passed in """ if self.inference: raise ValueError("Tried to train a model in inference mode.") xent_epochs = self.config.max_epochs trainer = training.Trainer( self.save_fn, max_epochs=xent_epochs, max_stalled_steps=self.config.max_stalled_steps) with tf.Session() as sess: if self.model_load: self.train_checkpoint.restore( self.model_load).assert_consumed().run_restore_ops() warnings.warn( "You are reloading a model for training. This feature" " is still not fully implemented. It restores the state" " of the model variables and optimizer but not the number" " of stalled steps, the validation cost record, or the" " state of the shuffled corpora") print("Restored model at {}".format(self.model_load)) else: sess.run(tf.global_variables_initializer()) training.training_loop( sess, self.model, trainer, self.data, self.train_feeds, self.infer_feeds, train_batch_size=self.config.train_batch_size, valid_batch_size=self.config.infer_batch_size, min_epochs_before_validation=1)
def main(): print("Loading data from pickle...") t = datetime.datetime.now() train_lines, charmap, inv_charmap = load_data_from_pickle("dubsmash") print("Finished in:", datetime.datetime.now() - t) args = {} args['lambda_'] = 10 args['n_critic_iters_per_generator_iter'] = 10 args['batch_size'] = 128 args['lr'] = 1e-4 args['adam_beta1'] = 0.5 args['adam_beta2'] = 0.9 args['iterations'] = 199000 args['continue_training'] = True args['netG_checkpoint'] = "Checkpoints/netG-10300012:39:32AM_12-05-20" args['netD_checkpoint'] = "Checkpoints/netD-10300012:39:32AM_12-05-20" training_loop(train_lines, charmap, inv_charmap, dataloader, args)
print("Please select an option below by entering the number:") print("1. Train using default settings in AirSim") print("2. Train using default settings without simulation") print("3. Train using custom settings") print("4. Evaluation Mode") print("5. View the results of the last training session in text UI") print("6. View the results of the last training session in AirSim") print("7. Change the environment settings") print("Enter Q at any time to quit") menu_in = input() if menu_in == "1": # train in airsim wait = input( "WARNING: Ensure your AirSim Unreal Environment is running") training.training_loop(True) elif menu_in == "2": # train without sim training.training_loop(False) elif menu_in == "3": # custom settings # if user has run custom settings, offer rerun if custom_set: if validation("Rerun with the previous custom settings? Y/N", False, "Y", "N") == 0: # run with last settings training.training_loop(is_airsim_in, reuse_in, blue_in, red_in, episodes_in, steps_in, paths_in) continue
def main_process(): tf.logging.set_verbosity(tf.logging.INFO) args = manage_arguments() output_dir = args.output_dir # define the output directory of the experiment output_dir += "/model-fc-{}-{}-{}-{}-{}-{}-{}".format( args.fc_size, args.num_fc, args.kernel_size, args.strides, args.pool_size, str(args.dropout_rate).replace('.', '_'), str(args.learning_rate).replace('.', '_') ) tf.gfile.MakeDirs(output_dir) # prepare the dataset if necessary if not os.path.isfile(args.data_dir + '/Train_Images.TFRecord'): tf.gfile.MakeDirs(args.data_dir) dataloader.prepare_datasets(args.data_dir) tf.enable_eager_execution() # define the train/valid dataloaders ds_train = dataloader.input_dataset_fn( batch_size=args.batch_size, image_file=args.data_dir + 'Train_Images.TFRecord', label_file=args.data_dir + 'Train_Labels.TFRecord', repeat=True, shuffle=True, drop_remainder=False, data_augmentation=True, ) steps_per_epoch = int(math.ceil(24287 / args.batch_size)) ds_eval = dataloader.input_dataset_fn( batch_size=args.batch_size, image_file=args.data_dir + 'Eval_Images.TFRecord', label_file=args.data_dir + 'Eval_Labels.TFRecord', repeat=False, shuffle=False, drop_remainder=False, data_augmentation=False, ) validation_steps = int(math.ceil(8095 / args.batch_size)) # define the model conv_list = [(2, 64), (2, 128), (3, 256)] dense_list = [args.fc_size] * args.num_fc dense_list.append(17) network_model = model.create_model( input_shape=(224, 224, 3), conv_list=conv_list, dense_list=dense_list, kernel_size=args.kernel_size, strides=args.strides, pool_size=args.pool_size, dropout_rate=args.dropout_rate, output_activation='sigmoid', layer_activation='relu' ) network_model.summary() # define the loss function, metrics, and optimizer loss_fn = tf.keras.losses.BinaryCrossentropy() metrics = [ tf.keras.metrics.BinaryAccuracy(), tf.keras.metrics.Precision(), tf.keras.metrics.Recall() ] best_metric_indicator_index = 0 optimizer = tf.keras.optimizers.Adam(learning_rate=args.learning_rate) # set up the mlflow experiment name and eventually retrieve the mlflow # run_id meta data. Afterward, log the params used in mlflow and start # the training process. At the end, log the returned metrics in mlflow # and return the metrics recorded within this instance of mlflow run. out_data = None mlflow.set_experiment(args.mlflow_experiment_name) run_id = training.get_mlflow_run_id(output_dir) run_metrics = None with mlflow.start_run(run_id=run_id) as run: run_id = run.info.run_id training.save_mlflow_run_id(run_id, output_dir) mlflow.log_param("pool_size", args.pool_size) mlflow.log_param("strides", args.strides) mlflow.log_param("kernel_size", args.kernel_size) mlflow.log_param("fc_size", args.fc_size) mlflow.log_param("num_fc", args.num_fc) mlflow.log_param("dropout_rate", args.dropout_rate) mlflow.log_param("learning_rate", args.learning_rate) mlflow.log_param("num_epochs", args.num_epochs) out_data = training.training_loop( network_model, optimizer, loss_fn, ds_train, args.num_epochs, steps_per_epoch, metrics=metrics, ds_eval=ds_eval, validation_steps=validation_steps, validation_freq=1, step_log_freq=20, checkpoint_path=output_dir, chkpt_freq=1, best_metric_coef=1.0, best_metric_indicator_index=best_metric_indicator_index, num_max_chkpts=args.num_max_chkpts, num_max_best_model=args.num_max_best_model, verbose=True ) if out_data is not None: for k in out_data.keys(): mlflow.log_metric(k, out_data[k]) run_metrics = out_data return run_metrics
def train(self, train_affect=False): """ Performs the training process given the configuration passed in :param bool train_affect: Whether to train using cross-entropy or affective loss """ if self.inference: raise ValueError("Tried to train a model in inference mode.") xent_epochs = self.config.max_epochs trainer = training.Trainer( self.save_fn, max_epochs=xent_epochs, max_stalled_steps=self.config.max_stalled_steps) with tf.Session() as sess: if self.model_load: warnings.warn( "You are reloading a model for training. This feature" " is still not fully implemented. It restores the state" " of the model variables and optimizer but not the number" " of stalled steps, the validation cost record, or the" " state of the shuffled corpora") self.train_checkpoint.restore( self.model_load).assert_consumed().run_restore_ops() print("Restored model at {}".format(self.model_load)) else: sess.run(tf.global_variables_initializer()) training.training_loop( sess, self.model, trainer, self.data, self.train_feeds, self.infer_feeds, train_batch_size=self.config.train_batch_size, valid_batch_size=self.config.infer_batch_size, min_epochs_before_validation=1) if train_affect: affect_epochs = (trainer.epochs_completed // 4) + 1 * (trainer.epochs_completed < 4) total_epochs = trainer.epochs_completed + affect_epochs train_feeds[self.model.train_affect] = True print( "Switching from cross-entropy to maximum affective content . . ." ) affect_trainer = training.Trainer( self.checkpoint_best, self.checkpoint_latest, save_fn, epochs_completed=trainer.epochs_completed, max_epochs=total_epochs, saver=trainer.saver, best_valid_cost=trainer.best_valid_cost) training.training_loop( sess, self.model, affect_trainer, self.data, self.train_feeds, self.valid_feeds, train_batch_size=self.config.train_batch_size, valid_batch_size=self.config.infer_batch_size)
save_path = './pokemon_results_' + type1 + '_' + type2 + '/' # Get data Xdset = dataloader.PokemonData(type=type1, data_path='./images_with_types') dataloader_X = DataLoader(Xdset, batch_size=8, shuffle=True) Ydset = dataloader.PokemonData(type=type2, data_path='./images_with_types') dataloader_Y = DataLoader(Ydset, batch_size=8, shuffle=True) # Get the model and train G_XtoY, G_YtoX, D_X, D_Y = network.create_model() losses, G_XtoY, G_YtoX, D_X, D_Y = training.training_loop( dataloader_X=dataloader_X, dataloader_Y=dataloader_Y, n_epochs=4000, G_XtoY=G_XtoY, G_YtoX=G_YtoX, D_X=D_X, D_Y=D_Y, lr=0.0005, beta1=0.5, beta2=0.999, save_path=save_path) # save the model checkpoint_dir = './Checkpoints/' G_XtoY_path = os.path.join(checkpoint_dir, 'G_XtoY.pkl') G_YtoX_path = os.path.join(checkpoint_dir, 'G_YtoX.pkl') D_X_path = os.path.join(checkpoint_dir, 'D_X.pkl') D_Y_path = os.path.join(checkpoint_dir, 'D_Y.pkl') torch.save(G_XtoY.state_dict(), G_XtoY_path) torch.save(G_YtoX.state_dict(), G_YtoX_path) torch.save(D_X.state_dict(), D_X_path)
def main(): """ Define training pipelines """ dataset_train, dataset_valid = generate_default_training_pipeline( tfr_path, channels, n_modes, validation_split, batch_size, shuffle_buffer, n_prefetch, cpu=False) """ Define model """ model = model_cnn_mlp(channels, nz, nx, n_modes, cpu=False) optimizer = tf.keras.optimizers.Adam(learning_rate=0.001) model.compile(loss='mean_squared_error', optimizer=optimizer, metrics=['mean_absolute_error']) model_loss = tf.keras.losses.MeanSquaredError() """ Training loop """ training_loop(dataset_train, dataset_valid, save_path, model_name, model, optimizer, model_loss, epochs) # model_name = f"Ret_flow-reconstruction_yp{yp_flow:03d}" # start_time = time.time() # train_loss = tf.metrics.Mean() # valid_loss = tf.metrics.Mean() # for epoch in range(1, epochs + 1): # train_loss.reset_states() # for (X_target, Y_target) in dataset_train: # pred_loss, mae = model.train_on_batch(X_target, Y_target) # train_loss.update_state(pred_loss) # for (X_target, Y_target) in dataset_valid: # valid_pred_loss, valid_mae = model.test_on_batch(X_target, Y_target) # valid_loss.update_state(valid_pred_loss) # end_time = time.time() # if epoch > 10: # model.optimizer.lr = 0.001 * tf.math.exp(0.1 * (10 - epoch)) # print(f'Epoch {epoch:04d}/{epochs:04d}, loss: {train_loss.result().numpy()}, val_loss: {valid_loss.result().numpy()}, elapsed time from start: {end_time - start_time}') # predictor_name = models_path + model_name + '_predictor.tf' # predictor.save(predictor_name) return
NUM_DRAWS_PHASES[0]) solver_params_copy = solver_params.copy() solver_params_copy.update({ 'save_at': SAVE_MODEL_AT_PHASES[0], }) fit_params_copy = fit_params.copy() fit_params_copy.update({ 'epochs': EPOCHS_PHASES[0], 'basic_only': True, 'single_annotator': annotator, }) training_loop(dataset, BATCH_SIZES, learning_rates, local_folder, EPOCHS_PHASES[0], solver_params_copy, fit_params_copy, phase_path=phase, annotator_path=annotator) if phase is 'pretraining': learning_rates = get_learning_rates(LR_INT[0], LR_INT[1], NUM_DRAWS_PHASES[1]) solver_params_copy = solver_params.copy() solver_params_copy.update({ 'save_at': SAVE_MODEL_AT_PHASES[1], }) fit_params_copy = fit_params.copy() fit_params_copy.update({ 'epochs': EPOCHS_PHASES[1],
f.close() dataset.remove_pseudo_labels() if METHOD == 'majority_voting': mv_labels = {data_item: Counter([label[0] for label in list(data[data_item].values())]).most_common(1)[0][0] for data_item in data} f = open(labels_path, "wb") pickle.dump(mv_labels, f) f.close() dataset.remove_pseudo_labels() if TRAINING and mode == 'train': sample_label_map = {} with open(labels_path, 'rb') as f: sample_label_map = pickle.load(f) dataset.use_custom_labels(sample_label_map) learning_rates = get_learning_rates( LR_INT[0], LR_INT[1], NUM_DRAWS) epochs = EPOCHS if USE_EPOCH_FACTOR: epochs = EPOCHS * epoch_factor fit_params = { 'return_f1': True, 'deep_randomization': DEEP_RANDOMIZATION, 'early_stopping_interval': EARLY_STOPPING_INTERVAL, 'epochs': epochs, 'pretrained_basic': False, 'basic_only': True, } training_loop(dataset, BATCH_SIZES, learning_rates, local_folder, epochs, solver_params, fit_params)