patience=4,
min_lr=0.000001,
cooldown=3,
verbose=1)
stop_on_nan = keras.callbacks.TerminateOnNaN()
# LR finder
if opt.find_lr:
# pre-train to avoid model being too far away from interesting range
history = model.fit_generator(gen_x_train,
epochs=2,
verbose=1,
callbacks=[clr])
lr_finder = LRFinder(model)
lr_finder.find_generator(gen_x_train, 0.00001, 1.0, 5)
lr_finder.plot_loss()
import pdb
pdb.set_trace()
# Run training
if not opt.notrain:
# Train classifier
history = model.fit_generator(
gen_x_train,
epochs=epochs,
verbose=1, # switch to 1 for more verbosity
callbacks=[early_stopping, clr,
stop_on_nan], #, reduce_lr], #, lr, reduce_lr],
# callbacks=[early_stopping, reduce_lr], #, lr, reduce_lr],
use_multiprocessing=True,
def main(batch_size: int = 24,
epochs: int = 384,
train_path: str = 'train',
val_path: str = 'val',
multi_gpu_weights=None,
weights=None,
workers: int = 8,
find_lr: bool = False):
keras_model = MobileDetectNetModel.complete_model()
keras_model.summary()
if weights is not None:
keras_model.load_weights(weights, by_name=True)
train_seq = MobileDetectNetSequence(train_path,
stage="train",
batch_size=batch_size)
val_seq = MobileDetectNetSequence(val_path,
stage="val",
batch_size=batch_size)
keras_model = keras.utils.multi_gpu_model(keras_model,
gpus=[0, 1],
cpu_merge=True,
cpu_relocation=False)
if multi_gpu_weights is not None:
keras_model.load_weights(multi_gpu_weights, by_name=True)
callbacks = []
def region_loss(classes):
def loss_fn(y_true, y_pred):
# Don't penalize bounding box errors when there is no object present
return 10 * classes * K.abs(y_pred - y_true)
return loss_fn
keras_model.compile(optimizer=SGD(),
loss=[
'mean_absolute_error',
region_loss(
keras_model.get_layer('classes').output),
'binary_crossentropy'
])
if find_lr:
from lr_finder import LRFinder
lr_finder = LRFinder(keras_model)
lr_finder.find_generator(train_seq,
start_lr=0.000001,
end_lr=1,
epochs=5)
lr_finder.plot_loss()
return
filepath = "weights-{epoch:02d}-{val_loss:.4f}-multi-gpu.hdf5"
checkpoint = ModelCheckpoint(filepath,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min')
callbacks.append(checkpoint)
sgdr_sched = SGDRScheduler(0.00001,
0.01,
steps_per_epoch=np.ceil(
len(train_seq) / batch_size),
mult_factor=1.5)
callbacks.append(sgdr_sched)
keras_model.fit_generator(
train_seq,
validation_data=val_seq,
epochs=epochs,
steps_per_epoch=np.ceil(len(train_seq) / batch_size),
validation_steps=np.ceil(len(val_seq) / batch_size),
callbacks=callbacks,
use_multiprocessing=True,
workers=workers,
shuffle=True)
