if pretrained_path is not None:
new_model.load_weights(pretrained_path)
# finetune the whole network together.
for layer in new_model.layers:
layer.trainable = True
sgd = keras.optimizers.SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)
new_model.compile(optimizer=sgd, loss=alpha_prediction_loss)
print(new_model.summary())
# Summarize then go!
num_cpu = get_available_cpus()
workers = int(round(num_cpu / 2))
# Final callbacks
callbacks = [tensor_board, model_checkpoint, early_stop, reduce_lr]
# Start Fine-tuning
new_model.fit_generator(train_gen(),
steps_per_epoch=num_train_samples // batch_size,
validation_data=valid_gen(),
validation_steps=num_valid_samples // batch_size,
epochs=epochs,
verbose=1,
callbacks=callbacks,
use_multiprocessing=True,
workers=workers
)
# rewrite the callback: saving through the original model and not the multi-gpu model.
model_checkpoint = MyCbk(model)
else:
new_model = build_model()
if pretrained_path is not None:
new_model.load_weights(pretrained_path)
loss_funcs = get_loss_funcs()
# sgd optimizer with lr multipliers
# lr_multipliers = get_lr_multipliers(new_model)
# multisgd = MultiSGD(lr=base_lr, momentum=momentum, decay=0.0, nesterov=True, lr_mult=lr_multipliers)
adam = keras.optimizers.Adam(lr=base_lr)
new_model.compile(optimizer=adam, loss='mean_squared_error')
print(new_model.summary())
# Final callbacks
callbacks = [tensor_board, model_checkpoint, early_stop, reduce_lr]
ensure_folder('models')
# Start Fine-tuning
new_model.fit_generator(train_gen(),
steps_per_epoch=num_train_samples // batch_size,
validation_data=valid_gen(),
validation_steps=num_valid_samples // batch_size,
epochs=epochs,
verbose=1,
callbacks=callbacks,
use_multiprocessing=False)