def run_bot(batch_sampling_method, window_size, stride, batch_size, num_training_steps, learning_rate, geometric_decay, conv_layers_separable, len_conv1_filters, num_conv1_features, num_conv2_features, num_fc1_neurons, model_ending, dropout_keep_prob) -> float: # TODO: remove hardcoding of args (metaclass? couldn't get args to unpack hparams = HParams(**inspect.getargvalues(inspect.currentframe())[3]) hparam_dict = hparams.values() print(hparam_dict) bot = crypto_bot.CryptoBot(hparams, set_params(), test=TEST, tuning=True) cost = -bot.get_value() master_entry = {} lowest_cost = 1.0 #with open('hparams_master.pickle', 'rb') as handle: # master_entry = pickle.load(handle) #master_entry[cost] = hparam_dict #with open('hparams_master.pickle'.format(basedir), 'wb') as handle: # pickle.dump(master_entry, handle, protocol=pickle.HIGHEST_PROTOCOL) # TODO: figure out another method of logging because this can't be a part of the class # if cost < self.lowest_cost: # self.lowest_cost = cost return cost
def _load_hparams(path): with open(os.path.join(path, 'hparams.json'), 'rb') as json_file: hparams_dict = { k.encode('utf-8'): v.encode('utf-8') if type(v) == unicode else v for k, v in json.load(json_file).iteritems() } hparams = HParams(**hparams_dict) hparams.set_hparam('data_dir', path) trainer_lib.add_problem_hparams(hparams, 'translate_mmt') # Removing dropout from HParams even on TRAIN mode for key in hparams.values(): if key.endswith("dropout"): setattr(hparams, key, 0.0) return hparams
def run_experiment(unused_argv): """Run the training experiment.""" hyperparameters_dict = FLAGS.__flags # Build the hyperparameters object params = HParams(**hyperparameters_dict) # Set the seeds np.random.seed(params.random_seed) tf.set_random_seed(params.random_seed) # Initialise the run config run_config = tf.contrib.learn.RunConfig() # Use JIT XLA session_config = tf.ConfigProto() if params.use_jit_xla: session_config.graph_options.optimizer_options.global_jit_level = ( tf.OptimizerOptions.ON_1) # Clean the model directory if os.path.exists(params.model_dir) and params.clean_model_dir: shutil.rmtree(params.model_dir) # Update the run config run_config = run_config.replace(tf_random_seed=params.random_seed) run_config = run_config.replace(model_dir=params.model_dir) run_config = run_config.replace(session_config=session_config) run_config = run_config.replace( save_checkpoints_steps=params.min_eval_frequency) # Output relevant info for inference ex.save_dict_json(d=params.values(), path=os.path.join(params.model_dir, 'params.dict'), verbose=True) ex.save_obj(obj=params, path=os.path.join(params.model_dir, 'params.pkl'), verbose=True) learn_runner.run(experiment_fn=ex.experiment_fn, run_config=run_config, schedule='train_and_evaluate', hparams=params)
def create_hparams(hparams_string=None, verbose=False): """Create model hyperparameters. Parse nondefault from given string.""" hparams = HParams( ################################ # General Parameters # ################################ logging_batch_index_perc=10, # Percentage of samples used from the full dataset between logging the loss for training and testing. start_with_test=True, # Determines if the model is tested first before any training loops. # The computed loss is also used to identify the best model so far. # Therefore, if this is False and use_best_as_final_model is True # the best model of the current training will be saved, which possibly # overrides an older better model. log_memory_consumption=True, epochs_per_test=1, # Number of training epochs before testing (NOTE that this includes the scheduler_type with epoch scheduling). networks_dir="nn", checkpoints_dir="checkpoints", epochs_per_checkpoint=1, # Number of epochs between checkpoints, 0 for no checkpoints at all. save_final_model=True, # Determines if the model is saved after training. use_best_as_final_model=True, # Substitutes the saved final model with the best of the current run. ################################ # Experiment Parameters # ################################ epochs=0, test_set_perc=0.05, # Percentage of samples taken from the given id_list in __init__ for testing. # Ignored when self.id_list_train is already set. Note that self.id_list_test must be set then as well. val_set_perc=0.05, # Percentage of samples taken from the given id_list in __init__ for validation. # Ignored when self.id_list_train is already set. Note that self.id_list_val should be set then as well. seed=1234, # Used to initialize torch, numpy, and random. If None, the id_list is not shuffled before taking test and validation set from it. fp16_run=False, # TODO: Not implemented. # distributed_run=False, # TODO: Find out how distributed run works. # dist_url="file://distributed.dpt", # cudnn_enabled=True, # cudnn_benchmark=False, use_gpu=False, num_gpus=1, # TODO: Change to num_devices. batch_first=False, # Note: This might not be implemented properly everywhere. variable_sequence_length_train=None, # Do samples in mini batches during training have variable length. variable_sequence_length_test=None, # Do samples in mini batches during testing have variable length. shuffle_train_set=True, # Shuffle in dataset to get mini batches. shuffle_val_set=False, # Shuffle in dataset to get mini batches. batch_size_train=1, batch_size_test=48, # batch_size_val=1, # TODO: Add again after finding all My* classes where it is missing. batch_size_benchmark=48, batch_size_synth=48, batch_size_gen_figure=48, dataset_num_workers_gpu=4, # Number of workers used in dataset when running on GPU(s). dataset_num_workers_cpu=0, # Number of workers used in dataset when running on CPU(s). dataset_pin_memory=True, dataset_load_async=True, teacher_forcing_in_test=False, # If True, the targets are also given to the model when running the test (needed for WaveNet). preload_next_batch_to_gpu=False, # If True loads the next batch to GPU while processing the current one. # This enhances GPU usage for the cost of memory, because two batches are loaded to the GPU. # TODO: This does not work yet, because cuda async does lazy loading. ################################ # Data Parameters # ################################ len_in_out_multiplier=1, out_dir=None, ################################ # Audio Parameters # ################################ # sampling_frequency=16000, # TODO: Unused? frame_size=5, # max_wav_value=32768.0, ################################ # Model Parameters # ################################ model_type=None, model_name=None, model_dir=None, # Explicitly set directory where model is stored, otherwise dir_out/networks_dir/. dropout=0.0, hidden_init=0.0, # Hidden state init value train_hidden_init=False, # Is the hidden state init value trainable # TODO: Unused? ################################ # Optimization Hyperparameters # ################################ loss_per_sample=False, # If True the loss is first averaged on each sample and then over the batch. # If False the loss is averaged over each frame in the whole batch (default). backward_retain_graph=False, # Determines if the gradient computation should do aggressive memory freeing. # Only needed when gradient computational graph is reused. optimiser_type="Adam", # "Adam", "SGD" TODO: more optimiser_args=dict(), # Set optimiser arguments. Preferred way to set learning rate: optimiser_args["lr"]=... use_saved_learning_rate=True, # Use the learning rate saved with a model after loading it. replace_inf_grads_by_zero=False, # Automatically substitute +/- inf gradients with zero during training. # dynamic_loss_scaling=True, exponential_moving_average=False, # TODO: Not implemented properly. exponential_moving_average_decay=0.9999, # Ignored when exponential_moving_average is False. scheduler_type="default", # "None", "Plateau", "Exponential","Noam", TODO: "Step", "Cyclic_cosine" scheduler_args=dict(), iterations_per_scheduler_step=None, # Number of training iterations after which the scheduler step function # is called with the current loss and total number of iterations as parameter. # If None the scheduler is not called. epochs_per_scheduler_step=None, # Number of training epochs after which the scheduler step function is # called with the current validation loss and total number of epochs. # When a model is loaded the epoch number continues from the epoch number stored in the model. grad_clip_norm_type=None, # If None no gradient clipping otherwise uses grad_clip_max_norm (small bias). grad_clip_max_norm=None, # Ignored if grad_clip_norm_type is None. grad_clip_thresh=None, # Clip absolute value of gradient (big bias). # Set optimiser or scheduler_type to ignore type configuration above. Used to try new implementations. optimiser=None, # Will be called with model parameters only. Set other parameters with partial. Example: partial(torch.optim.Adam, **args)). scheduler=None, # Will be called with optimiser only. Set other parameters with partial. Example: partial(ReduceLROnPlateau, **args)). ################################ # Synthesis Parameters # ################################ synth_vocoder="WORLD", # "WORLD", "r9y9wavenet_quantized_16k_world_feats" synth_ext="wav", # Extension of the output audio. synth_fs=16000, num_coded_sps=60, # Number of spectral features, currently always MGC. synth_dir=None, synth_acoustic_model_path=None, synth_file_suffix='', # do_post_filtering = False, # TODO: Merlin does some filtering before calling its vocoder. Possible implementation: https://github.com/r9y9/nnmnkwii/blob/master/nnmnkwii/postfilters/__init__.py synth_gen_figure=False, gen_figure_ext=".pdf", epochs_per_plot=0, # No plots per epoch with <= 0. # TODO: plot in run method each ... epochs. plot_per_epoch_id_list=None, # TODO: Id(s) in the dictionary which are plotted. ) if hparams_string: logging.info('Parsing command line hparams: %s', hparams_string) hparams.parse(hparams_string) if verbose: logging.info('Final parsed hparams: %s', hparams.values()) return hparams