def tuneHP(self, hyperModel, X_train, X_test, y_train, y_test, tuner_epochs=50, tuner_batch_size=10000, tuner_mode=0):
if tuner_mode == 0:
tuner = kt.Hyperband(hyperModel,
objective=kt.Objective("auc", direction="max"), # ['loss', 'auc', 'accuracy', 'val_loss', 'val_auc', 'val_accuracy']
max_epochs=200,
hyperband_iterations=3,
factor=3,
seed=seed_value,
directory='tuning',
project_name='model_hyperband_1',
overwrite=True)
elif tuner_mode == 1:
tuner = kt.BayesianOptimization(hyperModel,
objective='val_loss',
max_trials=100,
seed=seed_value,
directory='tuning',
project_name='model_bayesian_1',
overwrite=True)
elif tuner_mode == 2:
tuner = kt.RandomSearch(hyperModel,
objective='val_loss',
max_trials=1000,
seed=seed_value,
directory='tuning',
project_name='model_random_1',
overwrite=True)
else:
raise ValueError('Invalid tuner mode')
tuner.search(X_train, y_train, epochs=tuner_epochs, batch_size=tuner_batch_size, validation_data=(X_test, y_test), verbose=0)
# tuner.search(X_train, y_train, epochs=tuner_epochs, validation_data=(X_test, y_test), verbose=1)
best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]
print(tuner.search_space_summary())
return best_hps, tuner
def fit(self, x=None, y=None, validation_data=None, **kwargs):
# Initialize HyperGraph model
x = layer_utils.format_inputs(x, 'train_x')
y = layer_utils.format_inputs(y, 'train_y')
# TODO: Set the shapes only if they are not provided by the user when
# initiating the HyperHead or Block.
for x_input, input_node in zip(x, self.inputs):
input_node.shape = x_input.shape[1:]
for y_input, output_node in zip(y, self.outputs):
if len(y_input.shape) == 1:
y_input = np.reshape(y_input, y_input.shape + (1, ))
output_node.shape = y_input.shape[1:]
output_node.in_hypermodels[0].output_shape = output_node.shape
# Prepare the dataset
if validation_data is None:
(x, y), (x_val, y_val) = layer_utils.split_train_to_valid(x, y)
validation_data = x_val, y_val
self.tuner = kerastuner.RandomSearch(hypermodel=self,
objective='val_loss',
max_trials=self.max_trials,
directory=self.directory)
# TODO: allow early stop if epochs is not specified.
self.tuner.search(x=x, y=y, validation_data=validation_data, **kwargs)
def tuner_fn(working_dir: Text, train_data_pattern: Text,
eval_data_pattern: Text,
schema: schema_pb2.Schema) -> component.TunerFnResult:
"""Build the tuner using the Keras Tuner API.
Args:
working_dir: working dir for KerasTuner.
train_data_pattern: file pattern of training tfrecord data.
eval_data_pattern: file pattern of eval tfrecord data.
schema: Schema of the input data.
Returns:
A namedtuple contains the following:
- tuner: A KerasTuner that will be used for tuning.
- train_dataset: A tf.data.Dataset of training data.
- eval_dataset: A tf.data.Dataset of eval data.
"""
hparams = kerastuner.HyperParameters()
hparams.Choice('learning_rate', [1e-1, 1e-3])
hparams.Int('num_layers', 1, 5)
# TODO(jyzhao): support params, e.g., max_trials in user input config.
tuner = kerastuner.RandomSearch(_build_keras_model,
max_trials=5,
hyperparameters=hparams,
allow_new_entries=False,
objective='val_accuracy',
directory=working_dir,
project_name='iris')
return component.TunerFnResult(
tuner=tuner,
train_dataset=_make_input_dataset(train_data_pattern, schema, 10),
eval_dataset=_make_input_dataset(eval_data_pattern, schema, 10))
def tuner_builder_random():
global _project_name
tuner = kt.RandomSearch(hypermodel = builder , objective = "val_loss", max_trials=10, directory = f'./tuner_data/{_project_name}', )
return tuner
def build_search():
# Load model
hypermodel = HyperMNIST(10)
tuner = kt.RandomSearch(hypermodel,
objective='val_acc',
max_trials=5,
executions_per_trial=3,
directory='./tuner',
project_name='mnist')
tuner.search_space_summary()
return tuner
def tuner_fn(fn_args: FnArgs) -> TunerFnResult:
"""Build the tuner using the KerasTuner API.
Args:
fn_args: Holds args as name/value pairs.
- working_dir: working dir for tuning.
- train_files: List of file paths containing training tf.Example data.
- eval_files: List of file paths containing eval tf.Example data.
- train_steps: number of train steps.
- eval_steps: number of eval steps.
- schema_path: optional schema of the input data.
- transform_graph_path: optional transform graph produced by TFT.
Returns:
A namedtuple contains the following:
- tuner: A BaseTuner that will be used for tuning.
- fit_kwargs: Args to pass to tuner's run_trial function for fitting the
model , e.g., the training and validation dataset. Required
args depend on the above tuner's implementation.
"""
# RandomSearch is a subclass of kerastuner.Tuner which inherits from
# BaseTuner.
tuner = kerastuner.RandomSearch(_build_keras_model,
max_trials=6,
hyperparameters=_get_hyperparameters(),
allow_new_entries=False,
objective=kerastuner.Objective(
'val_sparse_categorical_accuracy',
'max'),
directory=fn_args.working_dir,
project_name='penguin_tuning')
transform_graph = tft.TFTransformOutput(fn_args.transform_graph_path)
train_dataset = _input_fn(fn_args.train_files,
fn_args.data_accessor,
transform_graph,
batch_size=_TRAIN_BATCH_SIZE)
eval_dataset = _input_fn(fn_args.eval_files,
fn_args.data_accessor,
transform_graph,
batch_size=_EVAL_BATCH_SIZE)
return TunerFnResult(tuner=tuner,
fit_kwargs={
'x': train_dataset,
'validation_data': eval_dataset,
'steps_per_epoch': fn_args.train_steps,
'validation_steps': fn_args.eval_steps
})
def test_tunable_false_hypermodel(tmp_dir):
def build_model(hp):
input_shape = (256, 256, 3)
inputs = tf.keras.Input(shape=input_shape)
with hp.name_scope('xception'):
# Tune the pooling of Xception by supplying the search space
# beforehand.
hp.Choice('pooling', ['avg', 'max'])
xception = kerastuner.applications.HyperXception(
include_top=False,
input_shape=input_shape,
tunable=False).build(hp)
x = xception(inputs)
x = tf.keras.layers.Dense(
hp.Int('hidden_units', 50, 100, step=10),
activation='relu')(x)
outputs = tf.keras.layers.Dense(NUM_CLASSES, activation='softmax')(x)
model = tf.keras.Model(inputs, outputs)
optimizer = tf.keras.optimizers.get(hp.Choice('optimizer', ['adam', 'sgd']))
optimizer.learning_rate = hp.Float(
'learning_rate', 1e-4, 1e-2, sampling='log')
model.compile(optimizer, loss='sparse_categorical_crossentropy')
return model
tuner = kerastuner.RandomSearch(
objective='val_loss',
hypermodel=build_model,
max_trials=4,
directory=tmp_dir)
x = np.random.random(size=(2, 256, 256, 3))
y = np.random.randint(0, NUM_CLASSES, size=(2,))
tuner.search(x, y, validation_data=(x, y), batch_size=2)
hps = tuner.oracle.get_space()
assert 'xception/pooling' in hps
assert 'hidden_units' in hps
assert 'optimizer' in hps
assert 'learning_rate' in hps
# Make sure no HPs from building xception were added.
assert len(hps.space) == 4
def tuner_fn(fn_args: FnArgs) -> TunerFnResult:
"""Build the tuner using the KerasTuner API.
Args:
fn_args: Holds args as name/value pairs.
- working_dir: working dir for tuning.
- train_files: List of file paths containing training tf.Example data.
- eval_files: List of file paths containing eval tf.Example data.
- train_steps: number of train steps.
- eval_steps: number of eval steps.
- schema_path: optional schema of the input data.
- transform_graph_path: optional transform graph produced by TFT.
Returns:
A namedtuple contains the following:
- tuner: A BaseTuner that will be used for tuning.
- fit_kwargs: Args to pass to tuner's run_trial function for fitting the
model , e.g., the training and validation dataset. Required
args depend on the above tuner's implementation.
"""
hp = kerastuner.HyperParameters()
# Defines search space.
hp.Choice('learning_rate', [1e-1, 1e-3])
hp.Int('num_layers', 1, 5)
# RandomSearch is a subclass of Keras model Tuner.
tuner = kerastuner.RandomSearch(
_build_keras_model,
max_trials=5,
hyperparameters=hp,
allow_new_entries=False,
objective='val_sparse_categorical_accuracy',
directory=fn_args.working_dir,
project_name='test')
schema = schema_pb2.Schema()
io_utils.parse_pbtxt_file(fn_args.schema_path, schema)
train_dataset = _input_fn(fn_args.train_files, schema)
eval_dataset = _input_fn(fn_args.eval_files, schema)
return TunerFnResult(
tuner=tuner,
fit_kwargs={
'x': train_dataset,
'validation_data': eval_dataset,
'steps_per_epoch': fn_args.train_steps,
'validation_steps': fn_args.eval_steps
})
def test_tunable_false_hypermodel(tmp_dir):
def build_model(hp):
input_shape = (256, 256, 3)
inputs = tf.keras.Input(shape=input_shape)
with hp.name_scope("xception"):
# Tune the pooling of Xception by supplying the search space
# beforehand.
hp.Choice("pooling", ["avg", "max"])
xception = kerastuner.applications.HyperXception(
include_top=False, input_shape=input_shape,
tunable=False).build(hp)
x = xception(inputs)
x = tf.keras.layers.Dense(hp.Int("hidden_units", 50, 100, step=10),
activation="relu")(x)
outputs = tf.keras.layers.Dense(NUM_CLASSES, activation="softmax")(x)
model = tf.keras.Model(inputs, outputs)
optimizer = tf.keras.optimizers.get(
hp.Choice("optimizer", ["adam", "sgd"]))
optimizer.learning_rate = hp.Float("learning_rate",
1e-4,
1e-2,
sampling="log")
model.compile(optimizer, loss="sparse_categorical_crossentropy")
return model
tuner = kerastuner.RandomSearch(objective="val_loss",
hypermodel=build_model,
max_trials=4,
directory=tmp_dir)
x = np.random.random(size=(2, 256, 256, 3))
y = np.random.randint(0, NUM_CLASSES, size=(2, ))
tuner.search(x, y, validation_data=(x, y), batch_size=2)
hps = tuner.oracle.get_space()
assert "xception/pooling" in hps
assert "hidden_units" in hps
assert "optimizer" in hps
assert "learning_rate" in hps
# Make sure no HPs from building xception were added.
assert len(hps.space) == 4
def test_tuning_correctness(tmp_dir):
tuner = kerastuner.RandomSearch(
seed=1337,
hypermodel=MockHyperModel(),
max_trials=2,
objective='loss',
executions_per_trial=2,
directory=tmp_dir,
)
tuner.search()
assert len(tuner.trials) == 2
m0_e0_epochs = [
float(np.average(x)) for x in MockHyperModel.mode_0_execution_0
]
m0_e1_epochs = [
float(np.average(x)) for x in MockHyperModel.mode_0_execution_1
]
m0_epochs = [(a + b) / 2 for a, b in zip(m0_e0_epochs, m0_e1_epochs)
] + [m0_e1_epochs[-1]]
m1_epochs = [float(np.average(x)) for x in MockHyperModel.mode_1_execution]
# Score tracking correctness
first_trial, second_trial = sorted(tuner.trials, key=lambda x: x.score)
assert first_trial.score == min(m0_epochs)
assert second_trial.score == min(m1_epochs)
state = tuner.get_state()
assert state['best_trial']['score'] == min(m0_epochs)
assert tuner._get_best_trials(1)[0] == first_trial
# Metrics tracking correctness
assert len(first_trial.executions) == 2
e0 = first_trial.executions[0]
e1 = first_trial.executions[1]
e0_per_epoch_metrics = e0.per_epoch_metrics.metrics_history['loss']
e1_per_epoch_metrics = e1.per_epoch_metrics.metrics_history['loss']
e0_per_batch_metrics = e0.per_batch_metrics.metrics_history['loss']
e1_per_batch_metrics = e1.per_batch_metrics.metrics_history['loss']
assert e0_per_epoch_metrics == m0_e0_epochs
assert e0_per_batch_metrics == MockHyperModel.mode_0_execution_0[-1]
assert e1_per_epoch_metrics == m0_e1_epochs
assert e1_per_batch_metrics == MockHyperModel.mode_0_execution_1[-1]
assert first_trial.averaged_metrics.metrics_history['loss'] == m0_epochs
def test_get_best_hyperparameters(tmp_dir):
hp1 = kerastuner.HyperParameters()
hp1.Fixed('a', 1)
trial1 = kerastuner.engine.trial.Trial(hyperparameters=hp1)
trial1.status = 'COMPLETED'
trial1.score = 10
hp2 = kerastuner.HyperParameters()
hp2.Fixed('a', 2)
trial2 = kerastuner.engine.trial.Trial(hyperparameters=hp2)
trial2.status = 'COMPLETED'
trial2.score = 9
tuner = kerastuner.RandomSearch(objective='val_accuracy',
hypermodel=build_model,
max_trials=2,
directory=tmp_dir)
tuner.oracle.trials = {trial1.trial_id: trial1, trial2.trial_id: trial2}
hps = tuner.get_best_hyperparameters()[0]
assert hps['a'] == 1
def setUp(self):
import Exercise06_03
self.exercises = Exercise06_03
self.file_url = 'https://raw.githubusercontent.com/PacktWorkshops/The-TensorFlow-Workshop/master/Chapter06/dataset/connect-4.csv'
self.data = pd.read_csv(self.file_url)
self.target = self.data.pop('class')
self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(
self.data, self.target, test_size=0.2, random_state=42)
np.random.seed(8)
tf.random.set_seed(8)
tuner = kt.RandomSearch(model_builder,
objective='val_accuracy',
max_trials=10)
tuner.search(self.X_train,
self.y_train,
validation_data=(self.X_test, self.y_test))
best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]
self.best_l2 = best_hps.get('l2')
self.model = tuner.hypermodel.build(best_hps)
def tuner_fn(fn_args: FnArgs) -> TunerFnResult:
"""Build the tuner using CloudTuner (KerasTuner instance).
Args:
fn_args: Holds args used to train and tune the model as name/value pairs. See
https://www.tensorflow.org/tfx/api_docs/python/tfx/components/trainer/fn_args_utils/FnArgs.
Returns:
A namedtuple contains the following:
- tuner: A BaseTuner that will be used for tuning.
- fit_kwargs: Args to pass to tuner's run_trial function for fitting the
model , e.g., the training and validation dataset. Required
args depend on the above tuner's implementation.
"""
custom_config_dict = _get_custom_config_dict(fn_args)
max_trials = custom_config_dict.get('max_trials', MAX_TRIALS)
transform_graph = tft.TFTransformOutput(fn_args.transform_graph_path)
# Construct a build_keras_model_fn that just takes hyperparams from get_hyperparameters as input.
build_keras_model_fn = functools.partial(
_build_keras_model, tf_transform_output=transform_graph)
# CloudTuner is a subclass of kerastuner.Tuner which inherits from BaseTuner.
#is_local_run = "custom_config" not in fn_args.custom_config
is_local_run = custom_config_dict.get("is_local_run", True)
absl.logging.info('is_local_run : %s' % is_local_run)
if is_local_run:
tuner = kerastuner.RandomSearch(build_keras_model_fn,
max_trials=max_trials,
hyperparameters=_get_hyperparameters(),
allow_new_entries=False,
objective=kerastuner.Objective(
'val_binary_accuracy', 'max'),
directory=fn_args.working_dir,
project_name='titanic_tuning')
else:
tuner = CloudTuner(
build_keras_model_fn,
project_id=fn_args.custom_config['ai_platform_training_args']
['project'],
region=fn_args.custom_config['ai_platform_training_args']
['region'],
max_trials=max_trials,
hyperparameters=_get_hyperparameters(),
objective=kerastuner.Objective('val_binary_accuracy', 'max'),
# objective=kerastuner.Objective('auc', 'min'),
directory=fn_args.working_dir)
train_dataset = _input_fn(fn_args.train_files,
fn_args.data_accessor,
transform_graph,
batch_size=TRAIN_BATCH_SIZE)
eval_dataset = _input_fn(fn_args.eval_files,
fn_args.data_accessor,
transform_graph,
batch_size=EVAL_BATCH_SIZE)
return TunerFnResult(tuner=tuner,
fit_kwargs={
'x': train_dataset,
'validation_data': eval_dataset,
'steps_per_epoch': fn_args.train_steps,
'validation_steps': fn_args.eval_steps
})
if loss_choice == "focal":
model.compile(loss=focal_loss(),
optimizer="adam",
metrics=['accuracy'])
elif loss_choice == "categorical_crossentropy":
model.compile(loss="categorical_crossentropy",
optimizer="adam",
metrics=['accuracy'])
return model
hypermodel = MyHyperModel(num_classes=15)
tuner = kt.RandomSearch(hypermodel,
objective='val_accuracy',
max_trials=100,
executions_per_trial=2,
directory='SCSSK',
project_name='Hyperparameter tuning')
tuner.search(inputs3d,
targets,
batch_size=1000,
epochs=50,
shuffle=True,
verbose=0,
validation_split=0.05)
models = tuner.get_best_models(num_models=20)
for i, model in enumerate(models):
model.save(f"best_3dsubm{i+1}.h5")
units=hp.Int(f'{i}_units', min_value=5, max_value=50, step=5),
activation='relu',
kernel_regularizer=tf.keras.regularizers.l2(0.0001)))
model.add(tf.keras.layers.Dense(units=(np.shape(target)[1])))
model.compile(optimizer=hp.Choice('optimiser',
['adam', 'adagrad', 'adadelta']),
loss='mean_squared_error')
return model
# Performing random search with objective of minimising loss function
tuner = kt.RandomSearch(build_model,
objective='val_loss',
max_trials=5,
executions_per_trial=3,
directory='optimization_folder')
tuner.search_space_summary()
tuner.search(x_train, y_train, validation_data=(x_test, y_test), epochs=10)
# finding the best hyperparameters for the neural network
best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]
# Building the model with the optimised hyperparameters
best_model = tuner.hypermodel.build(best_hps)
""" Call back is performed when the difference between the loss function of
traing data and test data is large. This also prevents overfitting
"""
monitor = tf.keras.callbacks.EarlyStopping(monitor='val_loss',
min_delta=1e-3,
print("[INFO] Instantiating a hyperband tuner object...")
tuner = kt.Hyperband(build_model,
objective="val_accuracy",
max_epochs=config.EPOCHS,
factor=3,
seed=42,
directory=config.OUTPUT_PATH,
project_name=args["tuner"])
# Check for random search tuner
elif args["tuner"] == "random":
# Instantiate the random search tuner object
print("[INFO] Instantiating a random search tuner object...")
tuner = kt.RandomSearch(build_model,
objective="val_accuracy",
max_trials=10,
seed=42,
directory=config.OUTPUT_PATH,
project_name=args["tuner"])
# Otherwise, use the bayesian optimization tuner
else:
# Instantiate the bayesian optimization tuner object
print("[INFO] Instantiating a bayesian optimization tuner object...")
tuner = kt.BayesianOptimization(build_model,
objective="val_accuracy",
max_trials=10,
seed=42,
directory=config.OUTPUT_PATH,
project_name=args["tuner"])
# Perform the hyperparameter search
model.add(Dropout(drop_rate))
model.add(Dense(5))
model.add(Activation('softmax'))
loss='sparse_categorical_crossentropy'
model.compile(optimizer=tf.keras.optimizers.Adam(hp.Float('learning_rate', 1e-4, 1e-2, sampling='log')),loss='sparse_categorical_crossentropy',metrics=['accuracy'])
print(model.summary())
return model
LOG_DIR = f"{int (time.time ())}"
tuner=kt.RandomSearch
tuner = kt.RandomSearch(
build_model,
objective='val_acc',
max_trials=2,
executions_per_trial=1,
directory=LOG_DIR)
tuner.search(x=X,
y=y,
verbose=2, # just slapping this here bc jupyter notebook. The console out was getting messy.
epochs=1,
batch_size=64,
validation_data=(X_test,y_test))
print (tuner.get_best_hyperparameters () [0].values)
print (tuner.results_summary ())
print (tuner.get_best_models () [0].summary ())
model=tuner.get_best_models()[0]
tf.keras.models.save_model(model,'mymodel.h5')
def main(_):
flags_obj = flags.FLAGS
setup_keras_tuner_config()
if flags_obj.distribution_strategy == 'tpu':
resolver = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(resolver)
tf.tpu.experimental.initialize_tpu_system(resolver)
strategy = tf.distribute.experimental.TPUStrategy(resolver)
strategy_scope = strategy.scope()
print("All devices: ", tf.config.list_logical_devices('TPU'))
elif flags_obj.distribution_strategy == 'gpu':
strategy = tf.distribute.MirroredStrategy()
strategy_scope = strategy.scope()
devices = ["device:GPU:%d" % i for i in range(flags_obj.num_gpus)]
print('NUMBER OF DEVICES: ', strategy.num_replicas_in_sync)
## identify data paths and sources
root_dir = flags_obj.data_dir # this is gs://<bucket>/folder where tfrecord are stored
file_pattern = "{}/image_classification_builder-train*.tfrecord*".format(
root_dir)
val_file_pattern = "{}/image_classification_builder-validation*.tfrecord*".format(
root_dir)
file_list = tf.io.gfile.glob(file_pattern)
all_files = tf.data.Dataset.list_files(tf.io.gfile.glob(file_pattern))
val_file_list = tf.io.gfile.glob(val_file_pattern)
val_all_files = tf.data.Dataset.list_files(
tf.io.gfile.glob(val_file_pattern))
train_all_ds = tf.data.TFRecordDataset(
all_files, num_parallel_reads=tf.data.experimental.AUTOTUNE)
val_all_ds = tf.data.TFRecordDataset(
val_all_files, num_parallel_reads=tf.data.experimental.AUTOTUNE)
# perform data engineering
dataset = train_all_ds.map(decode_and_resize)
val_dataset = val_all_ds.map(decode_and_resize)
#
BATCH_SIZE = flags_obj.train_batch_size
VALIDATION_BATCH_SIZE = flags_obj.validation_batch_size
dataset = dataset.map(normalize,
num_parallel_calls=tf.data.experimental.AUTOTUNE)
val_dataset = val_dataset.map(
normalize, num_parallel_calls=tf.data.experimental.AUTOTUNE)
val_ds = val_dataset.batch(VALIDATION_BATCH_SIZE)
AUTOTUNE = tf.data.experimental.AUTOTUNE
train_ds = prepare_for_training(dataset)
FINE_TUNING_CHOICE = True
NUM_CLASSES = 5
IMAGE_SIZE = (224, 224)
train_sample_size = 0
for raw_record in train_all_ds:
train_sample_size += 1
print('TRAIN_SAMPLE_SIZE = ', train_sample_size)
validation_sample_size = 0
for raw_record in val_all_ds:
validation_sample_size += 1
print('VALIDATION_SAMPLE_SIZE = ', validation_sample_size)
STEPS_PER_EPOCHS = train_sample_size // BATCH_SIZE
VALIDATION_STEPS = validation_sample_size // VALIDATION_BATCH_SIZE
"""Runs the hyperparameter search."""
if (flags_obj.tuner_type.lower() == 'BayesianOptimization'.lower()):
tuner = kt.BayesianOptimization(hypermodel=model_builder,
objective='val_accuracy',
tune_new_entries=True,
allow_new_entries=True,
max_trials=5,
directory=flags_obj.model_dir,
project_name='hp_tune_bo',
overwrite=True)
elif (flags_obj.tuner_type.lower() == 'RandomSearch'.lower()):
tuner = kt.RandomSearch(hypermodel=model_builder,
objective='val_accuracy',
tune_new_entries=True,
allow_new_entries=True,
max_trials=5,
directory=flags_obj.model_dir,
project_name='hp_tune_rs',
overwrite=True)
else:
tuner = kt.Hyperband(hypermodel=model_builder,
objective='val_accuracy',
max_epochs=3,
factor=2,
distribution_strategy=strategy,
directory=flags_obj.model_dir,
project_name='hp_tune_hb',
overwrite=True)
tuner.search(train_ds,
steps_per_epoch=STEPS_PER_EPOCHS,
validation_data=val_ds,
validation_steps=VALIDATION_STEPS,
epochs=3,
callbacks=[
tf.keras.callbacks.EarlyStopping('val_accuracy'),
ClearTrainingOutput()
])
# Get the optimal hyperparameters
best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]
print(f"""
The hyperparameter search is done.
The best number of nodes in the dense layer is {best_hps.get('units')}.
The best activation function in mid dense layer is {best_hps.get('dense_activation')}.
""")
# Build the model with the optimal hyperparameters and train it on the data
model = tuner.hypermodel.build(best_hps)
checkpoint_prefix = os.path.join(flags_obj.model_dir,
"best_hp_train_ckpt_{epoch}")
callbacks = [
tf.keras.callbacks.TensorBoard(
log_dir=os.path.join(flags_obj.model_dir, 'tensorboard_logs')),
tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_prefix,
save_weights_only=True)
]
model.fit(train_ds,
epochs=3,
steps_per_epoch=STEPS_PER_EPOCHS,
validation_data=val_ds,
validation_steps=VALIDATION_STEPS,
callbacks=callbacks)
logging.info('INSIDE MAIN FUNCTION user input model_dir %s',
flags_obj.model_dir)
# Save model trained with chosen HP in user specified bucket location
model_save_dir = os.path.join(flags_obj.model_dir, 'best_save_model')
model.save(model_save_dir)
batch_size=100,
verbose=2,
validation_split=0.2)
print('acc :', best_model.evaluate(x_test, y_test, verbose=0))
tuner_bayesian = kt.BayesianOptimization(
model_builder,
objective='val_acc',
max_trials=5,
directory='keras_tuner/bayesian', # 따로 만들어주지 않아도 알아서 만들어줌
project_name='mnist')
tuner_random_search = kt.RandomSearch(
model_builder,
objective='val_acc',
max_trials=5,
directory='keras_tuner/random_search', # 따로 만들어주지 않아도 알아서 만들어줌
project_name='mnist')
# 세가지중 hyperband 가 가장 잘찾는다 이거로 쓰면됨
tuner_hyperband = kt.Hyperband(
model_builder,
objective='val_loss',
max_trials=5,
directory='keras_tuner/hyperband', # 따로 만들어주지 않아도 알아서 만들어줌
project_name='mnist')
# optimize_hyper_parameter(tuner_bayesian)
# optimize_hyper_parameter(tuner_random_search)
optimize_hyper_parameter(tuner_hyperband)
lrn_rate = 1e-2
loss_choice = hp.Choice('loss_function', ["focal","categorical_crossentropy"])
if loss_choice == "focal":
model.compile(loss=focal_loss(),
optimizer=Adam(lrn_rate),
metrics=['accuracy'])
else:
model.compile(loss="categorical_crossentropy",
optimizer=Adam(lrn_rate),
metrics=['accuracy'])
return model
hypermodel = MyHyperModel(num_classes=15)
tuner = kt.RandomSearch(
hypermodel,
objective='val_accuracy',
max_trials=100,
executions_per_trial=3,
directory='SCSSK',
project_name='Dense Tuner')
tuner.search(inputs2d, targets,batch_size=1000, epochs=500,shuffle=True, verbose=0, validation_split = 0.05)
models=tuner.get_best_models(num_models=20)
for i,model in enumerate(models):
model.save(f"best_2dsubm{i+1}.h5")
tuner.results_summary()
def hyper_parameter_search(search_type='BO',
objective='mse',
seed=101,
max_trails=10,
directory=os.path.normpath('C:/'),
project_name='',
max_epochs=10,
factor=3,
epochs=10,
train_data=(),
val_data=()):
'''
Given the search type this method uses that optimization
method from keras tuner and finds the best parameters.
and returns the model with the best parameteres.
'''
search_type = search_type.upper()
if search_type == 'BO' or search_type == 'BAYESIANOPTIMIZATION':
tuner = kt.BayesianOptimization(model_build,
objective=objective,
seed=seed,
max_trials=max_trails,
directory=directory,
project_name=project_name)
elif search_type == 'RS' or search_type == 'RANDOMSEARCH':
tuner = kt.RandomSearch(model_build,
objective=objective,
seed=seed,
max_trials=max_trails,
directory=directory,
project_name=project_name)
elif search_type == 'HB' or search_type == 'HYPERBAND':
tuner = kt.Hyperband(model_build,
max_epochs=max_epochs,
objective=objective,
factor=factor,
directory=directory,
project_name=project_name)
else:
raise ValueError(
'The requested keras tuner search type doesnot exist\n')
tuner.search(train_data[0],
train_data[1],
epochs=epochs,
validation_data=(val_data[0], val_data[1]),
callbacks=[ClearTrainingOutput()],
verbose=1)
best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]
print(f"""
The hyperparameter search is complete. The optimal units
{best_hps.get('units')} and the optimal learning rate is
{best_hps.get('learning_rate')} and the optimal dropout
{best_hps.get('dropout')} and the optimal activation
{best_hps.get('dense_activation')}.""")
model = tuner.hypermodel.build(best_hps)
return model
def main(_):
flags_obj = flags.FLAGS
strategy = tf.distribute.MirroredStrategy()
data_dir = get_builtin_data()
train_gtr, validation_gtr = make_generators(data_dir, flags_obj)
idx_labels = map_labels(train_gtr)
"""Runs the hyperparameter search."""
if (flags_obj.tuner_type.lower() == 'BayesianOptimization'.lower()):
tuner = kt.tuners.BayesianOptimization(hypermodel=model_builder,
objective='val_accuracy',
tune_new_entries=True,
allow_new_entries=True,
max_trials=5,
directory=flags_obj.model_dir,
project_name='hp_tune_bo',
overwrite=True)
elif (flags_obj.tuner_type.lower() == 'RandomSearch'.lower()):
tuner = kt.RandomSearch(hypermodel=model_builder,
objective='val_accuracy',
tune_new_entries=True,
allow_new_entries=True,
max_trials=5,
directory=flags_obj.model_dir,
project_name='hp_tune_rs',
overwrite=True)
else:
tuner = kt.Hyperband(hypermodel=model_builder,
objective='val_accuracy',
max_epochs=3,
factor=2,
distribution_strategy=strategy,
directory=flags_obj.model_dir,
project_name='hp_tune_hb',
overwrite=True)
tuner.search(train_gtr,
steps_per_epoch=train_gtr.samples // train_gtr.batch_size,
validation_data=validation_gtr,
validation_steps=validation_gtr.samples //
validation_gtr.batch_size,
epochs=3,
callbacks=[tf.keras.callbacks.EarlyStopping('val_accuracy')])
# Get the optimal hyperparameters
best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]
print(f"""
The hyperparameter search is done.
The best number of nodes in the dense layer is {best_hps.get('units')}.
The optimal learning rate for the optimizer is {best_hps.get('learning_rate')}.
""")
# Build the model with the optimal hyperparameters and train it on the data
model = tuner.hypermodel.build(best_hps)
checkpoint_prefix = os.path.join(flags_obj.model_dir,
"best_hp_train_ckpt_{epoch}")
callbacks = [
tf.keras.callbacks.TensorBoard(
log_dir=os.path.join(flags_obj.model_dir, 'tensorboard_logs')),
tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_prefix,
save_weights_only=True)
]
steps_per_epoch = train_gtr.samples // train_gtr.batch_size
validation_steps = validation_gtr.samples // validation_gtr.batch_size
model.fit(train_gtr,
epochs=flags_obj.train_epoch_best,
steps_per_epoch=steps_per_epoch,
validation_data=validation_gtr,
validation_steps=validation_steps,
callbacks=callbacks)
logging.info('INSIDE MAIN FUNCTION user input model_dir %s',
flags_obj.model_dir)
# Save model trained with chosen HP in user specified bucket location
model_save_dir = os.path.join(flags_obj.model_dir, 'best_save_model')
model.save(model_save_dir)