def test_set_hp(): input_node = ak.Input(shape=(32, )) output_node = input_node output_node = ak.DenseBlock()(output_node) head = ak.RegressionHead() head.output_shape = (1, ) output_node = head(output_node) graph = graph_module.Graph( inputs=input_node, outputs=output_node, override_hps=[ hp_module.Choice("dense_block_1/num_layers", [6], default=6) ], ) hp = kerastuner.HyperParameters() graph.build(hp) for single_hp in hp.space: if single_hp.name == "dense_block_1/num_layers": assert len(single_hp.values) == 1 assert single_hp.values[0] == 6 return assert False
def _convert_to_kerastuner_hyperparameters( self, candidate_hparams: List[Dict[str, Any]] ) -> List[kerastuner.HyperParameters]: """Convert list of HSpace to a list of search space each with cardinality 1. Args: candidate_hparams: List of Dict of HParams with same keys. Returns: The list of hparams in the search space. """ if not candidate_hparams: raise ValueError( f'Expected a non-empty list of candidate_hparams. Got {candidate_hparams}' ) simple_search_space_list = [] for candidate_hparam in candidate_hparams: simple_search_space = kerastuner.HyperParameters() for key in candidate_hparam: simple_search_space.Choice(key, [candidate_hparam[key]]) simple_search_space_list.append(simple_search_space) return simple_search_space_list
def check_initial_hp(initial_hp, graph): hp = kerastuner.HyperParameters() hp.values = copy.copy(initial_hp) graph.build(hp) assert hp.values == initial_hp
def get_hyperparam() -> kerastuner.HyperParameters: hp = kerastuner.HyperParameters() hp.Choice(name='learning_rate', values=[1e-2, 1e-3], default=1e-2) return hp
# Defining Optimizer # Compiling the model Nadam = tf.keras.optimizers.Nadam(learning_rate=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-07, name='Nadam') # Compile model model.compile(loss='mse', optimizer=Nadam, metrics=['mape']) return model hp = kt.HyperParameters() model = build_model(hp) tuner = Hyperband(build_model, objective=kt.Objective('val_mape', direction='min'), max_epochs=2000, hyperband_iterations=2, directory='HyperBandTrials', project_name='PressureOpti_hl') # Defining the Early Stopping Function early_stopping_callback = EarlyStopping(monitor='val_mape', patience=500, min_delta=1e-4, restore_best_weights=True, mode='auto',
def test_time_series_input_node_build_no_error(): node = nodes.TimeseriesInput(lookback=2, shape=(32,)) hp = kerastuner.HyperParameters() input_node = node.build_node(hp) node.build(hp, input_node)
def main(): base_folder, return_list, test, timer, train, val = init_test() prefix = '' test_name = 'best_model_comparison2' print('Running best model comparison') csv_file = base_folder / 'temp' / f'{prefix}{test_name}.csv' for repetition in range(7): config = Config() configuration_name = ModelType.BASELINE config.training.model_type = 'baseline' config.training.cnn_repetition = 0 config.training.lstm_repetition = 1 config.training.dense_repetition = 0 config.training.model_size = 384 config.training.dropout = 0 config.training.initial_learning_rate = 0.001 config.training.batch_size = 128 config.training.label_smoothing = 0 config.training.mixup_alpha = 0 config.training.l2_regularization = 0 config.training.x_groups = [ ['prev_word_id'], ] config.training.y_groups = [ ['word_id'], ] hp = None eval_config(csv_file, timer, return_list, train, val, test, config, test_name, configuration_name, hp) config = Config() configuration_name = ModelType.DDC config.training.model_type = 'ddc' config.training.cnn_repetition = 0 config.training.lstm_repetition = 2 config.training.dense_repetition = 0 config.training.model_size = 512 config.training.dropout = 0.5 config.training.initial_learning_rate = 0.001 config.training.batch_size = 64 config.training.label_smoothing = 0 config.training.mixup_alpha = 0 config.training.l2_regularization = 0 config.training.x_groups = [['prev_word_id'], [ 'prev', 'next', ]] config.training.y_groups = [ ['word_id'], ] hp = None eval_config(csv_file, timer, return_list, train, val, test, config, test_name, configuration_name, hp) config = Config() configuration_name = 'Custom vec+id:id' config.training.model_type = ModelType.CUSTOM config.training.cnn_repetition = 2 config.training.lstm_repetition = 2 config.training.dense_repetition = 2 config.training.model_size = 512 config.training.dropout = 0.4 config.training.initial_learning_rate = 1e-2 config.training.batch_size = 128 config.training.label_smoothing = 0.5 config.training.mixup_alpha = 0.5 config.training.l2_regularization = 0 config.training.x_groups = [['prev_word_id', 'prev_word_vec'], DatasetConfig().categorical, DatasetConfig().audio, DatasetConfig().regression] config.training.y_groups = [ ['word_id'], ] hp = None eval_config(csv_file, timer, return_list, train, val, test, config, test_name, configuration_name, hp) config = Config() configuration_name = 'Custom vec+id:vec' config.training.model_type = ModelType.CUSTOM config.training.cnn_repetition = 2 config.training.lstm_repetition = 2 config.training.dense_repetition = 0 config.training.model_size = 512 config.training.dropout = 0.4 config.training.initial_learning_rate = 1e-2 config.training.batch_size = 128 config.training.label_smoothing = 0.5 config.training.mixup_alpha = 0.5 config.training.l2_regularization = 0 config.training.x_groups = [['prev_word_id', 'prev_word_vec'], DatasetConfig().categorical, DatasetConfig().audio, DatasetConfig().regression] config.training.y_groups = [ ['word_vec'], ] hp = None eval_config(csv_file, timer, return_list, train, val, test, config, test_name, configuration_name, hp) config = Config() configuration_name = 'MLSTM' config.training.model_type = ModelType.TUNE_MLSTM config.training.batch_size = 128 config.training.label_smoothing = 0.5 config.training.mixup_alpha = 0.5 config.training.x_groups = [['prev_word_id', 'prev_word_vec'], DatasetConfig().categorical, DatasetConfig().audio, DatasetConfig().regression] config.training.y_groups = [ ['word_id'], ] hp = kt.HyperParameters() fixed_params = { 'connections_0': 2, 'connections_1': 2, 'connections_2': 2, 'connections_3': 3, 'connections_4': 1, 'connections_5': 3, 'connections_6': 2, 'depth_0': 18, 'depth_1': 23, 'depth_2': 43, 'depth_3': 13, 'depth_4': 52, 'depth_5': 5, 'depth_6': 11, 'dropout_0': 0.25612932926324405, 'dropout_1': 0.1620424523625309, 'dropout_2': 0.4720468723284278, 'dropout_3': 0.43881829788147036, 'dropout_4': 0.44741780640383355, 'dropout_5': 0.3327191857714107, 'dropout_6': 0.1367707920005909, 'initial_learning_rate': 0.008, 'label_smoothing': 0.13716631669361445, 'lstm_layers': 3, 'width_0': 16, 'width_1': 9, 'width_2': 15, 'width_3': 16, 'width_4': 5, 'width_5': 11, 'width_6': 4, } for param, value in fixed_params.items(): hp.Fixed(param, value=value) eval_config(csv_file, timer, return_list, train, val, test, config, test_name, configuration_name, hp) pass
def test_hyperband_oracle_one_sweep_parallel(tmp_dir): hp = kt.HyperParameters() hp.Float('a', -100, 100) hp.Float('b', -100, 100) oracle = hyperband_module.HyperbandOracle(hyperparameters=hp, objective='score', hyperband_iterations=1, max_epochs=4, factor=2) oracle._set_project_dir(tmp_dir, 'untitled') # All round 0 trials from different brackets can be run # in parallel. round0_trials = [] for i in range(10): t = oracle.create_trial('tuner' + str(i)) assert t.status == 'RUNNING' round0_trials.append(t) assert len(oracle._brackets) == 3 # Round 1 can't be run until enough models from round 0 # have completed. t = oracle.create_trial('tuner10') assert t.status == 'IDLE' for t in round0_trials: oracle.update_trial(t.trial_id, {'score': 1}) oracle.end_trial(t.trial_id, 'COMPLETED') round1_trials = [] for i in range(4): t = oracle.create_trial('tuner' + str(i)) assert t.status == 'RUNNING' round1_trials.append(t) # Bracket 0 is complete as it only has round 0. assert len(oracle._brackets) == 2 # Round 2 can't be run until enough models from round 1 # have completed. t = oracle.create_trial('tuner10') assert t.status == 'IDLE' for t in round1_trials: oracle.update_trial(t.trial_id, {'score': 1}) oracle.end_trial(t.trial_id, 'COMPLETED') # Only one trial runs in round 2. round2_trial = oracle.create_trial('tuner0') assert len(oracle._brackets) == 1 # No more trials to run, but wait for existing brackets to end. t = oracle.create_trial('tuner10') assert t.status == 'IDLE' oracle.update_trial(round2_trial.trial_id, {'score': 1}) oracle.end_trial(round2_trial.trial_id, 'COMPLETED') t = oracle.create_trial('tuner10') assert t.status == 'STOPPED', oracle._current_sweep
def fit(self, x=None, y=None, epochs=None, callbacks=None, validation_split=0.2, validation_data=None, **kwargs): """Search for the best model and hyperparameters for the AutoModel. It will search for the best model based on the performances on validation data. # Arguments x: numpy.ndarray or tensorflow.Dataset. Training data x. y: numpy.ndarray or tensorflow.Dataset. Training data y. epochs: Int. The number of epochs to train each model during the search. If unspecified, by default we train for a maximum of 1000 epochs, but we stop training if the validation loss stops improving for 10 epochs (unless you specified an EarlyStopping callback as part of the callbacks argument, in which case the EarlyStopping callback you specified will determine early stopping). callbacks: List of Keras callbacks to apply during training and validation. validation_split: Float between 0 and 1. Defaults to 0.2. Fraction of the training data to be used as validation data. The model will set apart this fraction of the training data, will not train on it, and will evaluate the loss and any model metrics on this data at the end of each epoch. The validation data is selected from the last samples in the `x` and `y` data provided, before shuffling. This argument is not supported when `x` is a dataset. The best model found would be fit on the entire dataset including the validation data. validation_data: Data on which to evaluate the loss and any model metrics at the end of each epoch. The model will not be trained on this data. `validation_data` will override `validation_split`. The type of the validation data should be the same as the training data. The best model found would be fit on the training dataset without the validation data. **kwargs: Any arguments supported by keras.Model.fit. """ dataset, validation_data = self._prepare_data( x=x, y=y, validation_data=validation_data, validation_split=validation_split) # Initialize the hyper_graph. self._meta_build(dataset) # Initialize the Tuner. # The hypermodel needs input_shape, which can only be known after # preprocessing. So we preprocess the dataset once to get the input_shape, # so that the hypermodel can be built in the initializer of the Tuner, which # does not access the dataset. hp = kerastuner.HyperParameters() preprocess_graph, keras_graph = self.hyper_graph.build_graphs(hp) preprocess_graph.preprocess(dataset=dataset, validation_data=validation_data, fit=True) self.tuner = self.tuner(hyper_graph=self.hyper_graph, hypermodel=keras_graph, fit_on_val_data=self._split_dataset, overwrite=self.overwrite, objective=self.objective, max_trials=self.max_trials, directory=self.directory, seed=self.seed, project_name=self.name) # Process the args. if callbacks is None: callbacks = [] if epochs is None: epochs = 1000 if not any([ isinstance(callback, tf.keras.callbacks.EarlyStopping) for callback in callbacks ]): callbacks = callbacks + [ tf.keras.callbacks.EarlyStopping(patience=10) ] self.tuner.search(x=dataset, epochs=epochs, callbacks=callbacks, validation_data=validation_data, **kwargs)
def test_greedy_oracle_get_state_update_space_can_run(): oracle = greedy.GreedyOracle(objective="val_loss") oracle.set_state(oracle.get_state()) hp = kerastuner.HyperParameters() hp.Boolean("test") oracle.update_space(hp)
def test_lightgbm_classifier_block(): input_node = ak.Input() hp = kerastuner.HyperParameters() lgbm_classifier = ak.LightGBMClassifierBlock() output_node = lgbm_classifier.build(hp=hp, inputs=input_node) assert isinstance(output_node[0], node.Node)
def main(): base_folder, return_list, test, timer, train, val = init_test() seed = 43 # random, non-fine tuned seed tf.random.set_seed(seed) np.random.seed(seed) random.seed(seed) config = Config() config.dataset.storage_folder = base_folder / 'new_datasets' config.audio_processing.use_cache = True model_path = base_folder / 'temp' model_path.mkdir(parents=True, exist_ok=True) # The best found temperature is ~0.91 configuration_name = 'MLSTM' config.training.model_type = ModelType.TUNE_MLSTM config.training.batch_size = 128 config.training.label_smoothing = 0.5 config.training.mixup_alpha = 0.5 config.training.x_groups = [['prev_word_id', 'prev_word_vec'], DatasetConfig().categorical, DatasetConfig().audio, DatasetConfig().regression] config.training.y_groups = [ ['word_id'], ] hp = kt.HyperParameters() fixed_params = { 'connections_0': 2, 'connections_1': 2, 'connections_2': 2, 'connections_3': 3, 'connections_4': 1, 'connections_5': 3, 'connections_6': 2, 'depth_0': 18, 'depth_1': 23, 'depth_2': 43, 'depth_3': 13, 'depth_4': 52, 'depth_5': 5, 'depth_6': 11, 'dropout_0': 0.25612932926324405, 'dropout_1': 0.1620424523625309, 'dropout_2': 0.4720468723284278, 'dropout_3': 0.43881829788147036, 'dropout_4': 0.44741780640383355, 'dropout_5': 0.3327191857714107, 'dropout_6': 0.1367707920005909, 'initial_learning_rate': 0.008, 'label_smoothing': 0.13716631669361445, 'lstm_layers': 3, 'width_0': 16, 'width_1': 9, 'width_2': 15, 'width_3': 16, 'width_4': 5, 'width_5': 11, 'width_6': 4, } for param, value in fixed_params.items(): hp.Fixed(param, value=value) find_temperature_and_generate(base_folder, train, val, test, model_path, configuration_name, deepcopy(config), hp) # The best found temperature is ~0.71 configuration_name = 'vec:id' config.training.model_type = ModelType.CUSTOM config.training.cnn_repetition = 2 config.training.lstm_repetition = 2 config.training.dense_repetition = 0 config.training.model_size = 512 config.training.dropout = 0.4 config.training.initial_learning_rate = 1e-2 config.training.batch_size = 128 config.training.label_smoothing = 0.5 config.training.mixup_alpha = 0.5 config.training.l2_regularization = 0 config.training.x_groups = [[ 'prev_word_id', ], DatasetConfig().categorical, DatasetConfig().audio, DatasetConfig().regression] config.training.y_groups = [ ['word_id'], ] find_temperature_and_generate(base_folder, train, val, test, model_path, configuration_name, deepcopy(config)) # The best found temperature is ~0.147 configuration_name = 'vec:vec' config.training.model_type = ModelType.CUSTOM config.training.cnn_repetition = 2 config.training.lstm_repetition = 2 config.training.dense_repetition = 0 config.training.model_size = 512 config.training.dropout = 0.4 config.training.initial_learning_rate = 1e-2 config.training.batch_size = 128 config.training.label_smoothing = 0.5 config.training.mixup_alpha = 0.5 config.training.l2_regularization = 0 config.training.x_groups = [[ 'prev_word_vec', ], DatasetConfig().categorical, DatasetConfig().audio, DatasetConfig().regression] config.training.y_groups = [ ['word_vec'], ] find_temperature_and_generate(base_folder, train, val, test, model_path, configuration_name, deepcopy(config))
def _get_hyperparameters() -> kerastuner.HyperParameters: hp = kerastuner.HyperParameters() hp.Choice(H_SIZE, [5, 10]) return hp
def main(): args = get_args() folder = args["inp-folder"] del args["inp-folder"] loc = os.path.dirname(sys.argv[0]) data_loc = os.path.join(loc, "Data") log_dir = os.path.join(loc, "logs") log_search_dir = os.path.join( log_dir, datetime.datetime.now().strftime("%m%d-%H%M")) if not os.path.exists(log_dir): os.mkdir(log_dir) if not os.path.exists(log_search_dir): os.mkdir(log_search_dir) x_tr, x_te, y_tr, y_te, i_ref, o_ref = load_inp_folder( os.path.join(data_loc, folder)) print(i_ref) print(o_ref) start = time.time() hp = kt.HyperParameters() if len(y_tr.shape) > 1: hp.Fixed("out", y_tr.shape[1]) else: hp.Fixed("out", 1) hp.Fixed("norm", False) hp.Fixed("drop_out", 0.0) num_trials = 100 tuner = kt.tuners.BayesianOptimization( hp_estimator, "val_mae", num_trials, num_initial_points=10, directory=log_search_dir, hyperparameters=hp, distribution_strategy=tf.distribute.MirroredStrategy(), project_name="XFEL-ANN") rand_tuner = kt.tuners.RandomSearch( hp_estimator, "val_mae", num_trials, directory=log_search_dir, hyperparameters=hp, distribution_strategy=tf.distribute.MirroredStrategy(), project_name="XFEL-ANN") print(tuner.search_space_summary()) x_tr_v, x_val, y_tr_v, y_val = train_test_split(x_tr, y_tr) # get validation set tuner.search(x_tr_v, y_tr_v, epochs=3000, verbose=2, validation_data=(x_val, y_val), batch_size=1000, callbacks=[ tf.keras.callbacks.EarlyStopping(monitor='val_mae', patience=3), tf.keras.callbacks.TensorBoard('history') ]) rand_tuner.search(x_tr_v, y_tr_v, epochs=3000, verbose=2, validation_data=(x_val, y_val), batch_size=1000, callbacks=[ tf.keras.callbacks.EarlyStopping(monitor='val_mae', patience=3), tf.keras.callbacks.TensorBoard('history') ]) dur = time.time() - start print(f"Runtime: {dur}s") trials = tuner.oracle.get_best_trials(num_trials=num_trials) rand_trials = rand_tuner.oracle.get_best_trials(num_trials=num_trials) tr_score = [trial.score for trial in trials][::-1] ra_score = [trial.score for trial in rand_trials][::-1] plt.style.use(args["style-sheet"]) plt.figure(figsize=(7, 7)) plt.plot(tr_score, label="Bayesian Optimisation") plt.plot(ra_score, label="Random Search") plt.legend() label = time.time() plt.savefig("hp_" + args["inp-folder"] + "_" + str(label)) if args["refit-best"]: best_model = tuner.get_best_models()[0] best_model.fit(x_tr, y_tr, batch_size=1000, epochs=300) print(best_model.evaluate(x_te, y_te))
def get_tunable_hyper_parameters(self) -> kt.HyperParameters: hp = kt.HyperParameters() self.build(hp) return hp
def test_hyperband_oracle_one_sweep_parallel(tmp_dir): hp = kt.HyperParameters() hp.Float("a", -100, 100) hp.Float("b", -100, 100) oracle = hyperband_module.HyperbandOracle( hyperparameters=hp, objective=kt.Objective("score", "max"), hyperband_iterations=1, max_epochs=4, factor=2, ) oracle._set_project_dir(tmp_dir, "untitled") # All round 0 trials from different brackets can be run # in parallel. round0_trials = [] for i in range(10): t = oracle.create_trial("tuner" + str(i)) assert t.status == "RUNNING" round0_trials.append(t) assert len(oracle._brackets) == 3 # Round 1 can't be run until enough models from round 0 # have completed. t = oracle.create_trial("tuner10") assert t.status == "IDLE" for t in round0_trials: oracle.update_trial(t.trial_id, {"score": 1}) oracle.end_trial(t.trial_id, "COMPLETED") round1_trials = [] for i in range(4): t = oracle.create_trial("tuner" + str(i)) assert t.status == "RUNNING" round1_trials.append(t) # Bracket 0 is complete as it only has round 0. assert len(oracle._brackets) == 2 # Round 2 can't be run until enough models from round 1 # have completed. t = oracle.create_trial("tuner10") assert t.status == "IDLE" for t in round1_trials: oracle.update_trial(t.trial_id, {"score": 1}) oracle.end_trial(t.trial_id, "COMPLETED") # Only one trial runs in round 2. round2_trial = oracle.create_trial("tuner0") assert len(oracle._brackets) == 1 # No more trials to run, but wait for existing brackets to end. t = oracle.create_trial("tuner10") assert t.status == "IDLE" oracle.update_trial(round2_trial.trial_id, {"score": 1}) oracle.end_trial(round2_trial.trial_id, "COMPLETED") t = oracle.create_trial("tuner10") assert t.status == "STOPPED", oracle._current_sweep
def main(): base_folder, return_list, test, timer, train, val = init_test() prefix = 'hyper_' config = Config() model_path = base_folder / 'temp' find_model = True train_model = True eval_model = True if find_model: for model_type in [ModelType.TUNE_CLSTM, ModelType.TUNE_MLSTM]: train_seq = BeatmapSequence(df=train, is_train=True, config=config) val_seq = BeatmapSequence(df=val, is_train=False, config=config) test_seq = BeatmapSequence(df=test, is_train=False, config=config) # To search for a specific input:output combination, change `config` config.training.model_type = model_type tuner = kt.Hyperband( get_architecture_fn(config)(train_seq, False, config), objective=kt.Objective('val_avs_dist', direction='min'), hyperband_iterations=1, max_epochs=100, factor=4, directory=base_folder / 'temp' / 'hyper_search', project_name=f'{get_architecture_fn(config).__qualname__}', overwrite=False, # CAUTION! ) tuner.search_space_summary() callbacks = create_callbacks(train_seq, config) tuner.search( x=train_seq, validation_data=val_seq, callbacks=callbacks, epochs=60, verbose=2, workers=10, max_queue_size=16, use_multiprocessing=False, ) print(tuner.results_summary()) print(tuner.get_best_models(2)[0].summary()) print(tuner.get_best_models(2)[0].evaluate(test_seq)) if train_model: # Train specific huperparameters hp = kt.HyperParameters() fixed_params = { 'connections_0': 2, 'connections_1': 2, 'connections_2': 2, 'connections_3': 3, 'connections_4': 1, 'connections_5': 3, 'connections_6': 2, 'depth_0': 18, 'depth_1': 23, 'depth_2': 43, 'depth_3': 13, 'depth_4': 52, 'depth_5': 5, 'depth_6': 11, 'dropout_0': 0.25612932926324405, 'dropout_1': 0.1620424523625309, 'dropout_2': 0.4720468723284278, 'dropout_3': 0.43881829788147036, 'dropout_4': 0.44741780640383355, 'dropout_5': 0.3327191857714107, 'dropout_6': 0.1367707920005909, 'initial_learning_rate': 0.008, 'label_smoothing': 0.13716631669361445, 'lstm_layers': 3, 'width_0': 16, 'width_1': 9, 'width_2': 15, 'width_3': 16, 'width_4': 5, 'width_5': 11, 'width_6': 4, } for param, val in fixed_params.items(): hp.Fixed(param, value=val) model = get_architecture_fn(config)(train_seq, False, config)(hp) model.summary() tf.keras.utils.plot_model(model, to_file=base_folder / 'temp' / 'model_architecture.png', show_shapes=True) model.fit( x=train_seq, validation_data=val_seq, callbacks=callbacks, epochs=81, verbose=2, workers=10, max_queue_size=16, use_multiprocessing=False, ) model_path.mkdir(parents=True, exist_ok=True) save_model(model, model_path, train_seq, config, hp=hp) timer('Saved model', 5) if eval_model: stateful_model = tf.keras.models.load_model( model_path / 'stateful_model.keras', custom_objects={'Perplexity': Perplexity}) timer('Loaded stateful model', 5) input_folder = base_folder / 'human_beatmaps' / 'new_dataformat' output_folder = base_folder / 'testing' / 'generated_songs' song_codes_to_gen = list( x for x in test.index.to_frame()["name"].unique()[:5]) song_codes_to_gen = [ '133b', ] print(song_codes_to_gen) for song_code in song_codes_to_gen: beatmap_folder = input_folder / song_code print(beatmap_folder) generate_complete_beatmaps(beatmap_folder, output_folder, stateful_model, config) timer('Generated beatmaps', 5)
import tensorflow as tf #tf.keras.backend.set_floatx('float64') # settings related to dataset data_name = 'Pendulum' folder_name = 'exp2_best' num_data_train_files = 6 num_initial_conditions = 5000 # per training file # settings related to system num_steps = 51 input_dim = 2 delta_t = 0.02 hp = kerastuner.HyperParameters() # settings related to training epochs = 15 hp.Fixed('learning_rate', 10.0**(-3)) batch_size = 128 # settings related to network architecture hp.Fixed("omega_num_complex_pairs", 1) hp.Fixed("omega_num_real", 0) hp.Fixed("enc_dec_num_layers", 2) hp.Fixed('enc_dec_layer_width', 80) hp.Fixed("omega_num_layers", 1) hp.Fixed('omega_layer_width', 170)
def check_initial_hp(initial_hp, graph): hp = kerastuner.HyperParameters() for i in range(3): hp.values = copy.copy(initial_hp) graph.build(hp) assert len(set(initial_hp.keys()) - set(hp._hps.keys())) == 0
def test_static_space_errors(tmp_dir): def build_model_static(hp): inputs = keras.Input(shape=(INPUT_DIM,)) x = inputs for i in range(hp.get('num_layers')): x = keras.layers.Dense( units=hp.get('units_' + str(i)), activation='relu')(x) outputs = keras.layers.Dense(NUM_CLASSES, activation='softmax')(x) model = keras.Model(inputs, outputs) model.compile( optimizer=keras.optimizers.Adam( hp.Float('learning_rate', 1e-5, 1e-2)), loss='sparse_categorical_crossentropy', metrics=['accuracy']) return model hp = kerastuner.HyperParameters() hp.Int('num_layers', 1, 3, 1, default=2) hp.Int('units_0', 4, 6, 1, default=5) hp.Int('units_1', 4, 6, 1, default=5) with pytest.raises(RuntimeError, match='`allow_new_entries` is `False`'): tuner = kerastuner.tuners.RandomSearch( build_model_static, objective='val_accuracy', max_trials=2, directory=tmp_dir, hyperparameters=hp, allow_new_entries=False) tuner.search( x=TRAIN_INPUTS, y=TRAIN_TARGETS, epochs=2, validation_data=(VAL_INPUTS, VAL_TARGETS)) def build_model_static_invalid(hp): inputs = keras.Input(shape=(INPUT_DIM,)) x = inputs for i in range(hp.get('num_layers')): x = keras.layers.Dense( units=hp.get('units_' + str(i)), activation='relu')(x) outputs = keras.layers.Dense(NUM_CLASSES, activation='softmax')(x) model = keras.Model(inputs, outputs) model.compile( optimizer=keras.optimizers.Adam( hp.Float('learning_rate', 0.001, 0.008, 0.001)), loss='sparse_categorical_crossentropy', metrics=['accuracy']) return model with pytest.raises(RuntimeError, match='`allow_new_entries` is `False`'): tuner = kerastuner.tuners.RandomSearch( build_model_static_invalid, objective='val_accuracy', max_trials=2, directory=tmp_dir, hyperparameters=hp, allow_new_entries=False) tuner.search( x=TRAIN_INPUTS, y=TRAIN_TARGETS, epochs=2, validation_data=(VAL_INPUTS, VAL_TARGETS))
def build_GNNModel(hp=kt.HyperParameters(), metrics=True, loss_balance=1.0): '''Build model with hyper parameter object''' # small AMP (170k parameters) #hp.Choice('atom_feature_size', [32, 64, 128, 256], ordered=True, default=128) #hp.Choice('edge_feature_size', [1, 2, 3, 64], ordered=True, default=64) #hp.Choice('edge_hidden_size', [16, 32, 64, 128, 256], ordered=True, default=64) #hp.Int('mp_layers', 1, 6, step=1, default=4) #hp.Int('fc_layers', 2, 6, step=1, default=3) #hp.Int('edge_fc_layers', 2, 6, step=1, default=3) hp.Choice('atom_feature_size', [32, 64, 128, 256], ordered=True, default=256) hp.Choice('edge_feature_size', [1, 2, 3, 8, 64], ordered=True, default=3) hp.Choice('edge_hidden_size', [16, 32, 64, 128, 256], ordered=True, default=128) hp.Int('mp_layers', 1, 6, step=1, default=4) hp.Int('fc_layers', 2, 6, step=1, default=4) hp.Int('edge_fc_layers', 2, 6, step=1, default=4) hp.Choice('noise', [0.0, 0.025, 0.05, 0.1], ordered=True, default=0.025) hp.Choice('dropout', [True, False], default=True) hp.Fixed('rbf_low', 0.005) hp.Fixed('rbf_high', 0.20) hp.Choice('mp_activation', ['relu', 'softplus', 'tanh'], default='softplus') hp.Choice('fc_activation', ['relu', 'softplus'], default='softplus') # load peak standards standards = nmrdata.load_standards() model = GNNModel(hp, standards) # compile with MSLE (to treat vastly different label mags) optimizer = tf.keras.optimizers.Adam( hp.Choice('learning_rate', [1e-3, 5e-4, 1e-4, 1e-5], default=1e-4)) embeddings = nmrdata.load_embeddings() #label_idx = type_mask(r'.*\-H.*', embeddings, regex=True) label_idx = type_mask(r'.*', embeddings, regex=True) corr_loss = NameLoss(label_idx, s=loss_balance) loss = corr_loss label_idx = type_mask(r'.*\-H.*', embeddings, regex=True) h_rmsd = NameRMSD(label_idx, name='h_rmsd') label_idx = type_mask(r'.*\-N.*', embeddings, regex=True) n_rmsd = NameRMSD(label_idx, name='n_rmsd') label_idx = type_mask(r'.*\-C.*', embeddings, regex=True) c_rmsd = NameRMSD(label_idx, name='c_rmsd') label_idx = type_mask(r'.*\-H$', embeddings, regex=True) hn_rmsd = NameRMSD(label_idx, name='hn_rmsd') label_idx = type_mask(r'.*\-HA*', embeddings, regex=True) ha_rmsd = NameRMSD(label_idx, name='ha_rmsd') label_idx = type_mask(r'.*\-H.*', embeddings, regex=True) h_r = NameCorr(label_idx, name='h_r') label_idx = type_mask(r'.*\-N.*', embeddings, regex=True) n_r = NameCorr(label_idx, name='n_r') label_idx = type_mask(r'.*\-C.*', embeddings, regex=True) c_r = NameCorr(label_idx, name='c_r') label_idx = type_mask(r'.*\-H$', embeddings, regex=True) hn_r = NameCorr(label_idx, name='hn_r') label_idx = type_mask(r'.*\-HA.*', embeddings, regex=True) ha_r = NameCorr(label_idx, name='ha_r') label_idx = type_mask(r'.*\-HA.*', embeddings, regex=True) ha_r = NameCorr(label_idx, name='ha_r') ha_count = NameCount(label_idx, name='avg_ha_count') label_idx = type_mask(r'DFT.*', embeddings, regex=True) dft_r = NameCorr(label_idx, name='dft_r') dft_count = NameCount(label_idx, name='avg_dft_count') label_idx = type_mask(r'MB.*', embeddings, regex=True) mb_r = NameCorr(label_idx, name='mb_r') mb_count = NameCount(label_idx, name='avg_mb_count') model.compile(optimizer=optimizer, loss=loss, metrics=[ h_rmsd, n_rmsd, c_rmsd, hn_rmsd, ha_rmsd, h_r, n_r, c_r, hn_r, ha_r, ha_count, mb_r, mb_count, dft_r, dft_count ] if metrics else None) return model
def hyper_parameter_search(rnn_inputs, dense_inputs, targets: np.array, validation_split=0.2, max_epochs=15, selected_hyperparameters=None, search_title=None, model_name="AuthorAttributionModel"): """ Perfoms a hyper-parameter search on the network, writes it to the file system and return resulting tuner :param rnn_inputs: dictionary with inputs to RNN (shapes Nx?, N = #posts) :param dense_inputs: dictionary with inputs to dense branch (shapes Nx?, N = #posts) :param targets: one-hot encoded target users as numpy array with (shape NxM, N = #posts, M = #users) :param validation_split: percentage of samples that should be used for validation :param max_epochs: maximal epochs run by the tuner :param selected_hyperparameters: Only tune for selected parameters, or fix all to just get the tuner for building the model :param search_title: title of the search, used to write logs to file system :param model_name: name of the model :return: Keras tuner object """ if search_title is None: search_title = "search_" + datetime.datetime.now().strftime( "%Y-%m-%d_%H-%M-%S") num_train_samples = targets.shape[0] inputs, rnn_network_inputs, dense_network_inputs = prepare_inputs( rnn_inputs, dense_inputs, targets) pathlib.Path("hyperparams/" + search_title).mkdir(parents=True, exist_ok=True) num_users = targets.shape[1] num_dense_inputs = dense_network_inputs.shape[1] num_rnn_inputs = rnn_network_inputs.shape[1] num_rnn_inputs_dimension = rnn_network_inputs.shape[2] log_run_inputs(rnn_inputs, dense_inputs, num_users, num_dense_inputs, num_rnn_inputs, num_rnn_inputs_dimension, num_train_samples, search_title) classifier = AuthorClassifier(num_users, num_dense_inputs, num_rnn_inputs, num_rnn_inputs_dimension, num_train_samples, search_title, model_name) stop_callback = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=3) tensorboard_log_dir = "tensorboard_logs/" + search_title pathlib.Path(tensorboard_log_dir).mkdir(parents=True, exist_ok=True) hist_callback = tf.keras.callbacks.TensorBoard( log_dir=tensorboard_log_dir, histogram_freq=1, write_images=True, # embeddings_freq=1, write_graph=True) if selected_hyperparameters is None: tuner = kt.Hyperband(classifier, objective='val_accuracy', max_epochs=max_epochs, factor=3, directory='hyperparams', project_name=search_title) else: tunable_parameters = classifier.get_tunable_hyper_parameters() assert all(parameter in selected_hyperparameters.keys() for parameter in tunable_parameters.values.keys()), \ "not all parameters needed where provided, need: " + str(tunable_parameters.values.keys()) hp = kt.HyperParameters() for key in selected_hyperparameters.keys(): if type(selected_hyperparameters[key]) != list: hp.Fixed(key, selected_hyperparameters[key]) else: hp.Choice(key, selected_hyperparameters[key]) tuner = kt.Hyperband(classifier, hyperparameters=hp, objective='val_accuracy', max_epochs=max_epochs, factor=3, directory='hyperparams', project_name=search_title) tuner.search(inputs, targets, validation_split=validation_split, callbacks=[hist_callback, stop_callback]) print("Hyper-parameters search '" + search_title + "' completed. Top results:") tuner.results_summary(5) return tuner
def fit(self, x=None, y=None, validation_split=0, validation_data=None, **kwargs): """Search for the best model and hyperparameters for the AutoModel. It will search for the best model based on the performances on validation data. Args: x: numpy.ndarray or tensorflow.Dataset. Training data x. y: numpy.ndarray or tensorflow.Dataset. Training data y. validation_split: Float between 0 and 1. Fraction of the training data to be used as validation data. The model will set apart this fraction of the training data, will not train on it, and will evaluate the loss and any model metrics on this data at the end of each epoch. The validation data is selected from the last samples in the `x` and `y` data provided, before shuffling. This argument is not supported when `x` is a dataset. validation_data: Data on which to evaluate the loss and any model metrics at the end of each epoch. The model will not be trained on this data. `validation_data` will override `validation_split`. `validation_data` could be: - tuple `(x_val, y_val)` of Numpy arrays or tensors - tuple `(x_val, y_val, val_sample_weights)` of Numpy arrays - dataset or a dataset iterator For the first two cases, `batch_size` must be provided. For the last case, `validation_steps` must be provided. """ # Initialize HyperGraph model x = nest.flatten(x) y = nest.flatten(y) y = self._label_encoding(y) # TODO: Set the shapes only if they are not provided by the user when # initiating the HyperHead or Block. for y_input, output_node in zip(y, self.outputs): if len(y_input.shape) == 1: y_input = np.reshape(y_input, y_input.shape + (1, )) output_node.shape = y_input.shape[1:] output_node.in_hypermodels[0].output_shape = output_node.shape # Split the data with validation_split if (all([isinstance(temp_x, np.ndarray) for temp_x in x]) and all([isinstance(temp_y, np.ndarray) for temp_y in y]) and validation_data is None and validation_split): (x, y), (x_val, y_val) = utils.split_train_to_valid(x, y, validation_split) validation_data = x_val, y_val # TODO: Handle other types of input, zip dataset, tensor, dict. # Prepare the dataset x, y, validation_data = utils.prepare_preprocess(x, y, validation_data) self.preprocess(hp=kerastuner.HyperParameters(), x=x, y=y, validation_data=validation_data, fit=True) self.tuner = tuner.RandomSearch(hypermodel=self, objective='val_loss', max_trials=self.max_trials, directory=self.directory) # TODO: allow early stop if epochs is not specified. self.tuner.search(x=x, y=y, validation_data=validation_data, **kwargs)
plt.subplot(ploc) plt.xlabel(elem[n]) plt.plot(X_data[:, n], y_true, base[n], alpha=0.3, label=f"NH4 relative to {elem[n]} (observed)") plt.plot(X_data[:, n], y_pred, 'xk', alpha=0.3, label=f"NH4 relative to {elem[n]} (predicted)") plt.legend() # plt.savefig("./results.png", format='png') plt.show() if __name__ == '__main__': data = cropData(loadData()) xTest, xTrain, xVal, yTrain, yTest, yVal = splitData(data) (trainedModel, xTest, yTest, yPred) = trainModel(data) fullPlot(xTest, yTest, yPred) hyparams = kt.HyperParameters() hmodel = hyper_tune(hyparams, xTrain, yTrain, xVal, yVal) hmodel.fit(xTrain, yTrain) hyperPreds = hmodel.predict(xTest) print("The predictions coming back have shape", hyperPreds.shape) heval = hmodel.evaluate(xTest, yTest, batch_size=64) print("Tuned mean squared error:", heval) print(hmodel.summary()) fullPlot(xTest, yTest, hyperPreds)
def fit(self, x=None, y=None, epochs=None, callbacks=None, validation_split=0, validation_data=None, **kwargs): """Search for the best model and hyperparameters for the AutoModel. It will search for the best model based on the performances on validation data. # Arguments x: numpy.ndarray or tensorflow.Dataset. Training data x. y: numpy.ndarray or tensorflow.Dataset. Training data y. epochs: Int. The number of epochs to train each model during the search. If unspecified, by default we train for a maximum of 1000 epochs, but we stop training if the validation loss stops improving for 10 epochs (unless you specified an EarlyStopping callback as part of the callbacks argument, in which case the EarlyStopping callback you specified will determine early stopping). callbacks: List of Keras callbacks to apply during training and validation. validation_split: Float between 0 and 1. Fraction of the training data to be used as validation data. The model will set apart this fraction of the training data, will not train on it, and will evaluate the loss and any model metrics on this data at the end of each epoch. The validation data is selected from the last samples in the `x` and `y` data provided, before shuffling. This argument is not supported when `x` is a dataset. validation_data: Data on which to evaluate the loss and any model metrics at the end of each epoch. The model will not be trained on this data. `validation_data` will override `validation_split`. `validation_data` could be: - tuple `(x_val, y_val)` of Numpy arrays or tensors - tuple `(x_val, y_val, val_sample_weights)` of Numpy arrays - dataset or a dataset iterator For the first two cases, `batch_size` must be provided. For the last case, `validation_steps` must be provided. **kwargs: Any arguments supported by keras.Model.fit. """ dataset, validation_data = self._prepare_data( x=x, y=y, validation_data=validation_data, validation_split=validation_split) # Initialize the hypermodel. self._meta_build(dataset) self.hypermodel.set_io_shapes(dataset) # Build the hypermodel in tuner init. hp = kerastuner.HyperParameters() self.hypermodel.hyper_build(hp) self.hypermodel.preprocess(hp=kerastuner.HyperParameters(), dataset=dataset, validation_data=validation_data, fit=True) self.tuner = tuner.RandomSearch(hypermodel=self.hypermodel, objective='val_loss', max_trials=self.max_trials, directory=self.directory, seed=self.seed, project_name=self.name) self.hypermodel.clear_preprocessors() # Process the args. if callbacks is None: callbacks = [] if epochs is None: epochs = 1000 if not any([ isinstance(callback, tf.keras.callbacks.EarlyStopping) for callback in callbacks ]): callbacks = callbacks + [ tf.keras.callbacks.EarlyStopping(patience=10) ] self.tuner.search(x=dataset, epochs=epochs, callbacks=callbacks, validation_data=validation_data, **kwargs)
def test_time_series_input_node_deserialize_build_to_tensor(): node = ak.TimeseriesInput(shape=(32, ), lookback=2) node = nodes.deserialize(nodes.serialize(node)) node.shape = (32, ) output = node.build(kerastuner.HyperParameters()) assert isinstance(output, tf.Tensor)
def fit(self, x=None, y=None, validation_split=0, validation_data=None, **kwargs): """Search for the best model and hyperparameters for the AutoModel. It will search for the best model based on the performances on validation data. # Arguments x: numpy.ndarray or tensorflow.Dataset. Training data x. y: numpy.ndarray or tensorflow.Dataset. Training data y. validation_split: Float between 0 and 1. Fraction of the training data to be used as validation data. The model will set apart this fraction of the training data, will not train on it, and will evaluate the loss and any model metrics on this data at the end of each epoch. The validation data is selected from the last samples in the `x` and `y` data provided, before shuffling. This argument is not supported when `x` is a dataset. validation_data: Data on which to evaluate the loss and any model metrics at the end of each epoch. The model will not be trained on this data. `validation_data` will override `validation_split`. `validation_data` could be: - tuple `(x_val, y_val)` of Numpy arrays or tensors - tuple `(x_val, y_val, val_sample_weights)` of Numpy arrays - dataset or a dataset iterator For the first two cases, `batch_size` must be provided. For the last case, `validation_steps` must be provided. **kwargs: Any arguments supported by keras.Model.fit. """ dataset, validation_data = self.prepare_data( x=x, y=y, validation_data=validation_data, validation_split=validation_split) self._meta_build(dataset) self.hypermodel.set_io_shapes(dataset) hp = kerastuner.HyperParameters() self.hypermodel.hyper_build(hp) self.hypermodel.preprocess( hp=kerastuner.HyperParameters(), dataset=dataset, validation_data=validation_data, fit=True) self.tuner = tuner.RandomSearch( hypermodel=self.hypermodel, objective='val_loss', max_trials=self.max_trials, directory=self.directory, seed=self.seed, project_name=self.name) # TODO: allow early stop if epochs is not specified. self.tuner.search(x=dataset, validation_data=validation_data, **kwargs)