tune_new_entries=True,
objective="val_accuracy",
directory="test_dir",
project_name="case6",
)
tuner.search(x=x, y=y, epochs=5, validation_data=(val_x, val_y))
# """Case #7:
# - We predefine the search space
# - No unregistered parameters are allowed in `build`
# """
hp = HyperParameters()
hp.Choice("learning_rate", [1e-1, 1e-3])
hp.Int("num_layers", 2, 20)
def build_model(hp):
model = keras.Sequential()
model.add(layers.Flatten(input_shape=(28, 28)))
for i in range(hp.get("num_layers")):
model.add(layers.Dense(32, activation="relu"))
model.add(layers.Dense(10, activation="softmax"))
model.compile(
optimizer=keras.optimizers.Adam(hp.get("learning_rate")),
loss="sparse_categorical_crossentropy",
metrics=["accuracy"],
)
return model
def pretrain(self,inputs,outputs,iteration):
"""
Optimize the hyperparameters of the ANN.
Args:
inputs (np.array): All input data.
outputs (np.array): All output data.
iteration (int): Iteration number.
Returns:
best_hp (kerastuner.engine.hyperparameters.HyperParameters): Optimal hyperparameters.
Notes:
Optimization objective fixed on val_loss.
"""
print("### Performing Keras Tuner optimization of the ANN###")
# Select hyperparameters to tune
hp = HyperParameters()
valid_entries = ["activation","neurons","layers","learning_rate","regularization","sparsity"]
if not all([entry in valid_entries for entry in self["optimize"]]):
raise Exception("Invalid hyperparameters specified for optimization")
if "activation" in self["optimize"]:
hp.Choice("activation_function",["sigmoid","relu","swish","tanh"],default=self["activation"])
if "neurons" in self["optimize"]:
hp.Int("no_neurons",3,20,sampling="log",default=self["no_neurons"])
if "layers" in self["optimize"]:
hp.Int("no_hid_layers",1,6,default=self["no_layers"])
if "learning_rate" in self["optimize"]:
hp.Float("learning_rate",0.001,0.1,sampling="log",default=self["learning_rate"])
if "regularization" in self["optimize"]:
hp.Float("regularization",0.0001,0.01,sampling="log",default=self["kernel_regularizer"])
if "sparsity" in self["optimize"]:
hp.Float("sparsity",0.3,0.9,default=self["sparsity"])
no_hps = len(hp._space)
# In case none are chosen, only 1 run for fixed setting
if no_hps == 0:
hp.Fixed("no_neurons",self["no_neurons"])
# Load tuner
max_trials = self["max_trials"]*no_hps
path_tf_format = "logs"
time = datetime.now().strftime("%Y%m%d_%H%M")
tuner_args = {"objective":"val_loss","hyperparameters":hp,"max_trials":max_trials,
"executions_per_trial":self["executions_per_trial"],"directory":path_tf_format,
"overwrite":True,"tune_new_entries":False,"project_name":f"opt"}
## "overwrite":True,"tune_new_entries":False,"project_name":f"opt_{time}"}
if self["tuner"] == "random" or no_hps==0:
tuner = RandomSearchCV(self.build_hypermodel,**tuner_args)
elif self["tuner"] == "bayesian":
tuner = BayesianOptimizationCV(self.build_hypermodel,num_initial_points=3*no_hps,**tuner_args)
# Load callbacks and remove early stopping
callbacks_all = self.get_callbacks()
callbacks = [call for call in callbacks_all if not isinstance(call,keras.callbacks.EarlyStopping)]
# Optimize
tuner.search(inputs,outputs,epochs=self["tuner_epochs"],verbose=0,shuffle=True,
callbacks=callbacks,iteration_no=iteration)
# Retrieve and save best model
best_hp = tuner.get_best_hyperparameters()[0]
self.write_stats([best_hp.values],"ann_best_models")
# Make a table of tuner stats
scores = [tuner.oracle.trials[trial].score for trial in tuner.oracle.trials]
hps = [tuner.oracle.trials[trial].hyperparameters.values for trial in tuner.oracle.trials]
for idx,entry in enumerate(hps):
entry["score"] = scores[idx]
self.write_stats(hps,"ann_tuner_stats")
return best_hp
objective='val_accuracy')
tuner.search(x=x,
y=y,
epochs=5,
validation_data=(val_x, val_y))
# """Case #7:
# - We predefine the search space
# - No unregistered parameters are allowed in `build`
# """
hp = HyperParameters()
hp.Choice('learning_rate', [1e-1, 1e-3])
hp.Int('num_layers', 2, 20)
def build_model(hp):
model = keras.Sequential()
model.add(layers.Flatten(input_shape=(28, 28)))
for i in range(hp.get('num_layers')):
model.add(layers.Dense(32,
activation='relu'))
model.add(layers.Dense(10, activation='softmax'))
model.compile(
optimizer=keras.optimizers.Adam(hp.get('learning_rate')),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
return model
