def main(args):
# ensure directories are setup
dirs = [args.data_dir, args.ckpt_dir]
prepare_dirs(dirs)
# create base model
model = get_base_model()
# define params
params = {
# '0_dropout': ['uniform', 0.1, 0.5],
# '0_act': ['choice', ['relu', 'selu', 'elu', 'tanh', 'sigmoid']],
# '0_l2': ['log_uniform', 1e-1, 2],
# '2_act': ['choice', ['selu', 'elu', 'tanh', 'sigmoid']],
# '2_l1': ['log_uniform', 1e-1, 2],
'2_hidden': ['quniform', 512, 1000, 1],
'4_hidden': ['quniform', 128, 512, 1],
'all_act': ['choice', [[0], ['choice', ['selu', 'elu', 'tanh']]]],
'all_dropout': ['choice', [[0], ['uniform', 0.1, 0.5]]],
'all_batchnorm': ['choice', [0, 1]],
'all_l2': ['uniform', 1e-8, 1e-5],
'optim': ['choice', ["adam", "sgd"]],
'lr': ['uniform', 1e-3, 8e-3],
# 'batch_size': ['quniform', 32, 128, 1]
}
# instantiate hyperband object
hyperband = Hyperband(args, model, params)
# tune
results = hyperband.tune()
# dump results
save_results(results)
def create_and_fit_model():
# Create the base model
base_model = get_base_model(workspace, DATA_DIR)
base_model.summary()
assert base_model.output_shape == (None, 128)
# Create the head
head_input_shape = (128 * CONFIG.N_ARTIFACTS, )
head_model = create_head(head_input_shape, dropout=CONFIG.USE_DROPOUT)
# Implement artifact flow through the same model
model_input = layers.Input(shape=(CONFIG.IMAGE_TARGET_HEIGHT,
CONFIG.IMAGE_TARGET_WIDTH,
CONFIG.N_ARTIFACTS))
features_list = []
for i in range(CONFIG.N_ARTIFACTS):
features_part = model_input[:, :, :, i:i + 1]
features_part = base_model(features_part)
features_list.append(features_part)
concatenation = tf.keras.layers.concatenate(features_list, axis=-1)
assert concatenation.shape.as_list() == tf.TensorShape(
(None, 128 * CONFIG.N_ARTIFACTS)).as_list()
model_output = head_model(concatenation)
model = models.Model(model_input, model_output)
model.summary()
best_model_path = str(DATA_DIR / f'outputs/{MODEL_CKPT_FILENAME}')
checkpoint_callback = callbacks.ModelCheckpoint(filepath=best_model_path,
monitor="val_loss",
save_best_only=True,
verbose=1)
training_callbacks = [
AzureLogCallback(run),
create_tensorboard_callback(),
checkpoint_callback,
]
optimizer = get_optimizer(CONFIG.USE_ONE_CYCLE,
lr=CONFIG.LEARNING_RATE,
n_steps=len(paths_training) / CONFIG.BATCH_SIZE)
# Compile the model.
model.compile(optimizer=optimizer, loss="mse", metrics=["mae"])
# Train the model.
model.fit(dataset_training.batch(CONFIG.BATCH_SIZE),
validation_data=dataset_validation.batch(CONFIG.BATCH_SIZE),
epochs=CONFIG.EPOCHS,
callbacks=training_callbacks,
verbose=2)
if CONFIG.EPOCHS_TUNE:
# Un-freeze
for layer in base_model._layers:
layer.trainable = True
# Adjust learning rate
optimizer = tf.keras.optimizers.Nadam(
learning_rate=CONFIG.LEARNING_RATE_TUNE)
model.compile(optimizer=optimizer, loss="mse", metrics=["mae"])
logger.info('Start fine-tuning')
model.fit(dataset_training.batch(CONFIG.BATCH_SIZE),
validation_data=dataset_validation.batch(CONFIG.BATCH_SIZE),
epochs=CONFIG.EPOCHS_TUNE,
callbacks=training_callbacks,
verbose=2)
# Create dataset for validation.
# Note: No shuffle necessary.
paths = paths_validate
dataset = tf.data.Dataset.from_tensor_slices(paths)
dataset_norm = dataset.map(lambda path: tf_load_pickle(path),
tf.data.experimental.AUTOTUNE)
dataset_norm = dataset_norm.cache()
dataset_norm = dataset_norm.prefetch(tf.data.experimental.AUTOTUNE)
dataset_validation = dataset_norm
del dataset_norm
# Note: Now the datasets are prepared.
# Create the base model
base_model = get_base_model(workspace, DATA_DIR)
base_model.summary()
assert base_model.output_shape == (None, 128)
# Create the head
head_input_shape = (128 * CONFIG.N_ARTIFACTS, )
head_model = create_head(head_input_shape, dropout=CONFIG.USE_CROPOUT)
# Implement artifact flow through the same model
model_input = layers.Input(shape=(CONFIG.IMAGE_TARGET_HEIGHT,
CONFIG.IMAGE_TARGET_WIDTH,
CONFIG.N_ARTIFACTS))
features_list = []
for i in range(CONFIG.N_ARTIFACTS):
features_part = model_input[:, :, :, i:i + 1]