def evaluateBestParams(X_train, y_train):
"""
Evaluate the best hyperparameters found for the training dataset
:param X_train: numpy array, train data (independent variable)
:param y_train: numpy array, train data (dependent variable)
:return: optimized model, best found hyperparameters for the optimized model, tuned model
"""
tuner = kt.BayesianOptimization(
hypermodel=modelBuilder,
objective="val_loss",
max_trials=5,
num_initial_points=4,
alpha=0.0001,
beta=2.6,
seed=568,
# hyperparameters=None,
tune_new_entries=True,
allow_new_entries=True,
directory=directory,
project_name=folder_name)
## stop training early after reaching a certain value for validation
stop_early = keras.callbacks.EarlyStopping(monitor="val_loss", patience=5)
## Run hyperparameter search
tuner.search(X_train,
y_train,
epochs=50,
validation_split=0.2,
callbacks=[stop_early])
print("get_best_params:", tuner.get_best_hyperparameters(num_trials=1)[0])
print("get_best_model:", tuner.get_best_models())
return tuner.get_best_models(), tuner.get_best_hyperparameters(
num_trials=1)[0], tuner
def get_tuner(cfg_hypertune, model_builder, outdir, recreate, strategy):
import keras_tuner as kt
if cfg_hypertune["algorithm"] == "random":
print("Keras Tuner: Using RandomSearch")
cfg_rand = cfg_hypertune["random"]
return kt.RandomSearch(
model_builder,
objective=cfg_rand["objective"],
max_trials=cfg_rand["max_trials"],
project_name=outdir,
overwrite=recreate,
)
elif cfg_hypertune["algorithm"] == "bayesian":
print("Keras Tuner: Using BayesianOptimization")
cfg_bayes = cfg_hypertune["bayesian"]
return kt.BayesianOptimization(
model_builder,
objective=cfg_bayes["objective"],
max_trials=cfg_bayes["max_trials"],
num_initial_points=cfg_bayes["num_initial_points"],
project_name=outdir,
overwrite=recreate,
)
elif cfg_hypertune["algorithm"] == "hyperband":
print("Keras Tuner: Using Hyperband")
cfg_hb = cfg_hypertune["hyperband"]
return kt.Hyperband(
model_builder,
objective=cfg_hb["objective"],
max_epochs=cfg_hb["max_epochs"],
factor=cfg_hb["factor"],
hyperband_iterations=cfg_hb["iterations"],
directory=outdir + "/tb",
project_name="mlpf",
overwrite=recreate,
executions_per_trial=cfg_hb["executions_per_trial"],
distribution_strategy=strategy,
)
name='output')(dense3)
img_model = keras.Model(inputs, outputs)
hp_lr = hp.Float("lr", min_value=1e-4, max_value=1e-2, sampling="log")
img_model.compile(
optimizer=keras.optimizers.Adam(learning_rate=hp_lr),
loss='categorical_crossentropy' # TODO: is this a good choice?
,
metrics=['accuracy'])
return img_model
tuner = kt.BayesianOptimization(
hypermodel=build_model,
objective="val_accuracy",
max_trials=50,
executions_per_trial=2,
overwrite=True,
directory='temp',
project_name='testing',
)
tuner.search_space_summary()
tuner.search(train_generator, validation_data=valid_generator)
best_models = tuner.get_best_models(num_models=1) # get best k models
best_model = best_models[0]
# retrain the model using the best hyperparameters found:
best_hyperparams = tuner.get_best_hyperparameters()
final_model = build_model(best_hyperparams[0])
final_model.fit(train_generator,
def search(
self,
X: Sequence[str] = None,
y: Union[Sequence[Sequence[str]], Sequence[str]] = None,
*,
dataset: NLPDataset = None,
validation_X: Sequence[str] = None,
validation_y: Union[Sequence[Sequence[str]], Sequence[str]] = None,
validation_dataset: NLPDataset = None,
batch_size: int = 128,
n_epochs: int = 10,
optimizer: str = "adam",
optimizer_kwargs: Optional[Dict[str, Any]] = None,
learning_rate: float = 1e-3,
weight_decay: float = 0.0,
clip: Optional[float] = 5.0,
learning_rate_update_factor: float = 0.5,
learning_rate_update_epochs: int = 10,
learning_rate_warmup_steps: int = 0,
enable_early_stopping: bool = False,
early_stopping_patience: Optional[int] = None,
early_stopping_min_delta: float = 0.0,
early_stopping_use_best_epoch: bool = False,
early_stopping_monitor: int = 2, # 1 for loss, 2 for metric
checkpoint: Optional[str] = None,
log_file: Optional[str] = None,
verbose: int = 2,
distribute_strategy: Optional[tf.distribute.Strategy] = None,
max_trials: int = 20,
executions_per_trial: int = 3,
search_result_directory: Optional[str] = None,
) -> None:
training_dataset = self.temp_model.prepare_dataset(X, y, dataset)
assert (
validation_X is None or validation_y is None
) or validation_dataset is None, "No validation set is provided."
validation_dataset = self.temp_model.prepare_dataset(
validation_X, validation_y, validation_dataset)
training_tf_dataset = training_dataset.batchify(batch_size)
validation_tf_dataset = validation_dataset.batchify(batch_size,
shuffle=False)
monitor = "val_loss"
monitor_direction = "min"
if early_stopping_monitor == 2 and self.temp_model.get_monitor():
monitor = self.temp_model.get_monitor()
monitor_direction = "max"
has_validation_dataset = validation_tf_dataset is not None
if checkpoint is not None:
if has_validation_dataset:
checkpoint = os.path.join(
checkpoint, self.model_type._get_model_filename(epoch=0))
else:
checkpoint = os.path.join(
checkpoint, self.model_type._get_model_filename_template())
callbacks = default_supervised_model_callbacks(
learning_rate_update_factor=learning_rate_update_factor,
learning_rate_update_epochs=learning_rate_update_epochs,
learning_rate_warmup_steps=learning_rate_warmup_steps,
use_weight_decay=weight_decay > 0,
has_validation_dataset=has_validation_dataset,
enable_early_stopping=enable_early_stopping,
early_stopping_monitor=monitor,
early_stopping_monitor_direction=monitor_direction,
early_stopping_patience=early_stopping_patience,
early_stopping_min_delta=early_stopping_min_delta,
early_stopping_use_best_epoch=early_stopping_use_best_epoch,
checkpoint=checkpoint,
log_file=log_file,
)
optimizer_kwargs = optimizer_kwargs or dict()
optimizer_parameters = {
"learning_rate": learning_rate,
"weight_decay": weight_decay,
**optimizer_kwargs,
}
if clip is not None:
optimizer_parameters["clipnorm"] = clip
model_builder = create_model_builder(
self.model_type,
self.model_args,
self.model_kwargs,
optimizer,
optimizer_parameters,
distribute_strategy=distribute_strategy,
)
objective = kt.Objective(monitor, monitor_direction)
directory = None
project_name = None
if search_result_directory is not None:
search_result_directory = search_result_directory.rstrip(".\\/")
directory = os.path.dirname(search_result_directory)
project_name = os.path.basename(search_result_directory)
if directory == project_name:
project_name = "default"
self.tuner = kt.BayesianOptimization(
model_builder,
objective=objective,
hyperparameters=self.hyper_parameters,
max_trials=max_trials,
# executions_per_trial=executions_per_trial,
directory=directory,
project_name=project_name,
distribution_strategy=distribute_strategy,
overwrite=False,
)
self.tuner.search(
training_tf_dataset,
epochs=n_epochs,
validation_data=validation_tf_dataset,
callbacks=callbacks,
verbose=verbose,
)