def _build_model_or_tuner(self, name):
"""
function to switch between defining model or tuner (if hp exist)
:param name: name of model
:return: self
"""
if self.hp is None:
_model = self._build(hp=None)
if self.model_list is None:
self.model_list = [{
"model_name": name,
"tuner": None,
"model": _model
}]
else:
self.model_list.append({
"model_name": name,
"tuner": None,
"model": _model
})
else:
tuner_kwargs = {
"hypermodel": self._build,
"objective": "val_loss",
"max_trials": 10,
"max_epochs": 100,
"project_name": "TS_RNN_tuner_log",
"directory": self.save_dir,
"overwrite": True,
"hyperparameters": self.hp
}
tuner_kwargs.update(self.kwargs)
if self.tuner == "RandomSearch":
del tuner_kwargs['max_epochs']
_tuner = RandomSearch(**tuner_kwargs)
elif self.tuner == "Hyperband":
del tuner_kwargs['max_trials']
_tuner = Hyperband(**tuner_kwargs)
elif self.tuner == "BayesianOptimization":
del tuner_kwargs['max_epochs']
_tuner = BayesianOptimization(**tuner_kwargs)
else:
_tuner = BayesianOptimization(**tuner_kwargs)
if self.model_list is None:
self.model_list = [{
"model_name": name,
"tuner": _tuner,
"model": None
}]
else:
self.model_list.append({
"model_name": name,
"tuner": _tuner,
"model": None
})
return self
def get_tuner(opDir, method, max_trials):
hp = HyperParameters()
hp.Int('n_sp_hidden_lyrs', min_value=1, max_value=3, step=1)
hp.Choice('sp_hidden_nodes', [16, 32, 64, 128])
hp.Int('n_mu_hidden_lyrs', min_value=1, max_value=3, step=1)
hp.Choice('mu_hidden_nodes', [16, 32, 64, 128])
hp.Int('n_smr_hidden_lyrs', min_value=1, max_value=3, step=1)
hp.Choice('smr_hidden_nodes', [16, 32, 64, 128])
hp.Choice('classification_loss_sp', ['binary_crossentropy', 'hinge'])
hp.Choice('classification_loss_mu', ['binary_crossentropy', 'hinge'])
# loss_weights = []
# alpha = np.arange(0.1,0.8,0.1)
# beta = 1-alpha
# for i in range(len(beta)):
# gamma = np.arange(0.1, beta[i]-0.1, 0.1)
# for j in range(len(gamma)):
# beta_i = beta[i] - gamma[j]
# loss_weights.append([alpha[i], beta_i, gamma[j]])
# loss_weights = np.round(loss_weights,1).tolist()
# hp.Choice('loss_weights', loss_weights)
hp.Fixed('TR_STEPS', PARAMS['TR_STEPS'])
misc.print_model_summary(opDir + '/model_summary.txt',
get_Lemaire_MTL_model(hp))
if method == 'RandomSearch':
tuner = RandomSearch(
get_Lemaire_MTL_model,
hyperparameters=hp,
objective='val_loss',
max_trials=max_trials,
executions_per_trial=2,
overwrite=False,
directory=opDir,
project_name='B3_MTL_architecture_tuning_non_causal',
tune_new_entries=True,
allow_new_entries=True,
)
elif method == 'BayesianOptimization':
tuner = BayesianOptimization(
get_Lemaire_MTL_model,
hyperparameters=hp,
objective='val_loss',
max_trials=max_trials,
executions_per_trial=2,
overwrite=False,
directory=opDir,
project_name='B3_MTL_architecture_tuning_non_causal',
tune_new_entries=True,
allow_new_entries=True,
)
return tuner
def _tune(self):
self.tuner = RandomSearch(
self.build_estimator,
objective="val_loss",
max_trials=self.num_of_trials,
executions_per_trial=3,
seed=self.seed,
)
# TODO how to define make trials
# if self.tuning_method == 'default':
# self.tuner = Hyperband(
# self.build_estimator,
# objective="val_loss",
# max_epochs=self.epochs,
# hyperband_iterations=5,
# project_name="untitled_project"
# )
# These lines are strictly for de-bugging, do not use in production.
# If we wish to linearly regress to set baseline, should make separate script, as this is not ML
# self.tuner = RandomSearch(
# self.baseline_model,
# objective="val_loss",
# max_trials=1,
# executions_per_trial=1,
# seed=self.seed,
# )
self.tuner.search(self.train_features,
self.train_target,
epochs=self.epochs,
validation_data=(self.val_features, self.val_target),
callbacks=[TensorBoard('untitled_project')])
self.params = self.tuner.get_best_hyperparameters(num_trials=1)[0]
self.estimator = self.tuner.hypermodel.build(self.params)
self.tune_score = None # TODO get tune score
rmtree("untitled_project")
def get_tuner(opDir, method, max_trials):
hp = HyperParameters()
hp.Int('kernel_size', min_value=3, max_value=19, step=2) # 9
hp.Int('Nd', min_value=3, max_value=8, step=1) # 6
hp.Int('nb_stacks', min_value=3, max_value=10, step=1) # 8
hp.Int('n_layers', min_value=1, max_value=4, step=1) # 4
hp.Choice('n_filters', [8, 16, 32]) # 3
hp.Boolean('skip_some_connections') # 2
hp.Fixed('TR_STEPS', PARAMS['TR_STEPS'])
if method == 'RandomSearch':
tuner = RandomSearch(
get_Lemaire_model,
hyperparameters=hp,
objective='val_loss',
max_trials=max_trials,
executions_per_trial=2,
overwrite=False,
directory=opDir,
project_name='B3_architecture_tuning_non_causal',
tune_new_entries=True,
allow_new_entries=True,
)
elif method == 'BayesianOptimization':
tuner = BayesianOptimization(
get_Lemaire_model,
hyperparameters=hp,
objective='val_loss',
max_trials=max_trials,
executions_per_trial=2,
overwrite=False,
directory=opDir,
project_name='B3_architecture_tuning_non_causal',
tune_new_entries=True,
allow_new_entries=True,
)
return tuner
class HyperTuneKeras:
def __init__(self,
train_features,
train_target,
param_grid,
num_of_trials=100,
seed=None,
validation_percent=0.1,
epochs=8):
self.seed = seed
self.epochs = epochs
self.param_grid = param_grid
train_percent = 1 - validation_percent
self.train_features, self.val_features, \
self.train_target, self.val_target = train_test_split(train_features,
train_target,
train_size=train_percent,
random_state=seed)
self.num_of_trials = num_of_trials
self._tune()
def build_estimator(self, hp):
model = Sequential()
for i in range(
hp.Choice('number_of_hidden_layers',
values=self.param_grid['n_hidden'])):
model.add(
Dense(
kernel_initializer='normal',
units=hp.Choice(f"number_of_neurons_in_hidden_layer_{i}",
values=self.param_grid['n_neuron']),
activation='relu',
))
if i < 4:
model.add(
Dropout(
hp.Choice(f"dropout_rate",
values=self.param_grid['drop'])), )
model.add(Dense(1))
model.compile(
optimizer=Adam(learning_rate=hp.Choice(
"learning_rate", values=self.param_grid['learning_rate'])),
loss="mse",
metrics=["mse", "mae"],
)
return model
def baseline_model(self, hp):
model = Sequential()
model.add(Dense(self.train_features.shape[1]))
model.add(Dense(1))
model.compile(
optimizer=Adam(learning_rate=hp.Choice(
"learning_rate", values=self.param_grid['learning_rate'])),
loss="mse",
metrics=["mse", "mae"],
)
return model
def _tune(self):
self.tuner = RandomSearch(
self.build_estimator,
objective="val_loss",
max_trials=self.num_of_trials,
executions_per_trial=3,
seed=self.seed,
)
# TODO how to define make trials
# if self.tuning_method == 'default':
# self.tuner = Hyperband(
# self.build_estimator,
# objective="val_loss",
# max_epochs=self.epochs,
# hyperband_iterations=5,
# project_name="untitled_project"
# )
# These lines are strictly for de-bugging, do not use in production.
# If we wish to linearly regress to set baseline, should make separate script, as this is not ML
# self.tuner = RandomSearch(
# self.baseline_model,
# objective="val_loss",
# max_trials=1,
# executions_per_trial=1,
# seed=self.seed,
# )
self.tuner.search(self.train_features,
self.train_target,
epochs=self.epochs,
validation_data=(self.val_features, self.val_target),
callbacks=[TensorBoard('untitled_project')])
self.params = self.tuner.get_best_hyperparameters(num_trials=1)[0]
self.estimator = self.tuner.hypermodel.build(self.params)
self.tune_score = None # TODO get tune score
rmtree("untitled_project")
set `overwrite=True` to start a new search and ignore any previous results.
Available tuners are `RandomSearch`, `BayesianOptimization` and `Hyperband`.
**Note:** the purpose of having multiple executions per trial is to reduce
results variance and therefore be able to more accurately assess the
performance of a model. If you want to get results faster, you could set
`executions_per_trial=1` (single round of training for each model
configuration).
"""
tuner = RandomSearch(
build_model,
objective="val_accuracy",
max_trials=3,
executions_per_trial=2,
overwrite=True,
directory="my_dir",
project_name="helloworld",
)
"""
You can print a summary of the search space:
"""
tuner.search_space_summary()
"""
Then, start the search for the best hyperparameter configuration.
The call to `search` has the same signature as `model.fit()`.
"""
tuner.search(x_train, y_train, epochs=2, validation_data=(x_val, y_val))
# target = xr.concat((target.sel(time=slice(None , '2009-12-15')),
# target.sel(time=slice('2010-01-21', None))), dim='time')
##################
# Set up the tuner
##################
n_lev = len(predictor[height_dim])
hypermodel = MyHyperModel(n_inputs=n_lev)
tuner = RandomSearch(
hypermodel,
objective="val_loss",
max_trials=25,
executions_per_trial=3,
overwrite=False,
directory=home + "/Desktop/KerasTuning/",
project_name="project1",
)
################################################################
# Chunk the whole dataset into training, validation and test set
################################################################
n_samples = len(predictor)
n_trainsamples = round(0.8 * n_samples)
n_valsamples = n_samples - n_trainsamples
n_testsamples = round(0.1 * n_samples)
all_indices = np.arange(n_samples)
# Import the Cifar10 dataset.
NUM_CLASSES = 10
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
y_train = to_categorical(y_train, NUM_CLASSES)
y_test = to_categorical(y_test, NUM_CLASSES)
# Import an hypertunable version of Resnet.
hypermodel = HyperResNet(input_shape=x_train.shape[1:], classes=NUM_CLASSES)
# Initialize the hypertuner: we should find the model that maximixes the
# validation accuracy, using 40 trials in total.
tuner = RandomSearch(
hypermodel,
objective="val_accuracy",
max_trials=40,
project_name="cifar10_resnet",
directory="test_directory",
)
# Display search overview.
tuner.search_space_summary()
# Performs the hypertuning.
tuner.search(x_train, y_train, epochs=10, validation_split=0.1)
# Show the best models, their hyperparameters, and the resulting metrics.
tuner.results_summary()
# Retrieve the best model.
best_model = tuner.get_best_models(num_models=1)[0]
# Build model
model = keras.Model(inputs, outputs)
model.compile(
optimizer=Adam(lr), loss="categorical_crossentropy", metrics=["accuracy"]
)
return model
# Initialize the tuner by passing the `build_model` function
# and specifying key search constraints: maximize val_acc (objective),
# and the number of trials to do. More efficient tuners like UltraBand() can
# be used.
tuner = RandomSearch(
build_model,
objective="val_accuracy",
max_trials=TRIALS,
project_name="hello_world_tutorial_results",
)
# Display search space overview
tuner.search_space_summary()
# Perform the model search. The search function has the same signature
# as `model.fit()`.
tuner.search(
x_train, y_train, batch_size=128, epochs=EPOCHS, validation_data=(x_val, y_val)
)
# Display the best models, their hyperparameters, and the resulting metrics.
tuner.results_summary()
def create_tuner():
hypermodel = MyHyperModel(classes=10)
tuner = RandomSearch(hypermodel, objective="val_accuracy",
max_trials=3, overwrite=True, directory="my_dir",
project_name="helloworld")
return tuner