patience=4, min_lr=0.000001, cooldown=3, verbose=1) stop_on_nan = keras.callbacks.TerminateOnNaN() # LR finder if opt.find_lr: # pre-train to avoid model being too far away from interesting range history = model.fit_generator(gen_x_train, epochs=2, verbose=1, callbacks=[clr]) lr_finder = LRFinder(model) lr_finder.find_generator(gen_x_train, 0.00001, 1.0, 5) lr_finder.plot_loss() import pdb pdb.set_trace() # Run training if not opt.notrain: # Train classifier history = model.fit_generator( gen_x_train, epochs=epochs, verbose=1, # switch to 1 for more verbosity callbacks=[early_stopping, clr, stop_on_nan], #, reduce_lr], #, lr, reduce_lr], # callbacks=[early_stopping, reduce_lr], #, lr, reduce_lr], use_multiprocessing=True,
def main(batch_size: int = 24, epochs: int = 384, train_path: str = 'train', val_path: str = 'val', multi_gpu_weights=None, weights=None, workers: int = 8, find_lr: bool = False): keras_model = MobileDetectNetModel.complete_model() keras_model.summary() if weights is not None: keras_model.load_weights(weights, by_name=True) train_seq = MobileDetectNetSequence(train_path, stage="train", batch_size=batch_size) val_seq = MobileDetectNetSequence(val_path, stage="val", batch_size=batch_size) keras_model = keras.utils.multi_gpu_model(keras_model, gpus=[0, 1], cpu_merge=True, cpu_relocation=False) if multi_gpu_weights is not None: keras_model.load_weights(multi_gpu_weights, by_name=True) callbacks = [] def region_loss(classes): def loss_fn(y_true, y_pred): # Don't penalize bounding box errors when there is no object present return 10 * classes * K.abs(y_pred - y_true) return loss_fn keras_model.compile(optimizer=SGD(), loss=[ 'mean_absolute_error', region_loss( keras_model.get_layer('classes').output), 'binary_crossentropy' ]) if find_lr: from lr_finder import LRFinder lr_finder = LRFinder(keras_model) lr_finder.find_generator(train_seq, start_lr=0.000001, end_lr=1, epochs=5) lr_finder.plot_loss() return filepath = "weights-{epoch:02d}-{val_loss:.4f}-multi-gpu.hdf5" checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min') callbacks.append(checkpoint) sgdr_sched = SGDRScheduler(0.00001, 0.01, steps_per_epoch=np.ceil( len(train_seq) / batch_size), mult_factor=1.5) callbacks.append(sgdr_sched) keras_model.fit_generator( train_seq, validation_data=val_seq, epochs=epochs, steps_per_epoch=np.ceil(len(train_seq) / batch_size), validation_steps=np.ceil(len(val_seq) / batch_size), callbacks=callbacks, use_multiprocessing=True, workers=workers, shuffle=True)