def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save, valid_generator, config):
early_stop = EarlyStopping(
monitor = 'loss',
min_delta = 0.01,
patience = 5,
mode = 'min',
verbose = 1
)
checkpoint = CustomModelCheckpoint(
model_to_save = model_to_save,
filepath = saved_weights_name + '-epoch_{epoch:02d}-val_loss_{val_loss:.04f}.h5',
monitor = 'val_loss',
verbose = 1,
save_best_only = True,
mode = 'min',
period = 1
)
my_checkpoint = My_Checkpoint(
model_to_save=model_to_save,
valid_generator = valid_generator,
config = config
)
only_one_checkpoint = CustomModelCheckpoint(
model_to_save=model_to_save,
filepath=saved_weights_name + '.h5',
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min',
period=1
)
reduce_on_plateau = ReduceLROnPlateau(
monitor = 'loss',
factor = 0.1,
patience = 2,
verbose = 1,
mode = 'min',
epsilon = 0.01,
cooldown = 0,
min_lr = 0
)
# makedirs(tensorboard_logs)
# tensorboard = CustomTensorBoard(
# log_dir = tensorboard_logs,
# write_graph = True,
# write_images = True,
# )
return [early_stop, checkpoint, only_one_checkpoint, reduce_on_plateau, my_checkpoint]
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save):
makedirs(tensorboard_logs)
early_stop = EarlyStopping(monitor='loss',
min_delta=0.01,
patience=5,
mode='min',
verbose=1)
checkpoint = CustomModelCheckpoint(
model_to_save=model_to_save,
filepath=saved_weights_name, # + '{epoch:02d}.h5',
monitor='loss',
verbose=1,
save_best_only=True,
mode='min',
period=1)
checkpoint2 = CustomModelCheckpoint(
model_to_save=model_to_save,
filepath="watermark_val.h5", # + '{epoch:02d}.h5',
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='auto',
period=1)
checkpoint3 = CustomModelCheckpoint(
model_to_save=model_to_save,
filepath="watermark_valacc.h5", # + '{epoch:02d}.h5',
monitor='val_acc',
verbose=1,
save_best_only=True,
mode='auto',
period=1)
reduce_on_plateau = ReduceLROnPlateau(monitor='loss',
factor=0.1,
patience=2,
verbose=1,
mode='min',
epsilon=0.01,
cooldown=0,
min_lr=0)
tensorboard = CustomTensorBoard(
log_dir=tensorboard_logs,
write_graph=True,
write_images=True,
)
return [
early_stop, checkpoint, checkpoint2, checkpoint3, reduce_on_plateau,
tensorboard
]
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save,
valid_generator, labels):
makedirs(tensorboard_logs)
early_stop = EarlyStopping(monitor='val_loss',
min_delta=0.01,
patience=7,
mode='min',
verbose=1)
checkpoint = CustomModelCheckpoint(
model_to_save=model_to_save,
filepath=saved_weights_name, # + '{epoch:02d}.h5',
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min',
period=1)
reduce_on_plateau = ReduceLROnPlateau(monitor='loss',
factor=0.1,
patience=2,
verbose=1,
mode='min',
epsilon=0.01,
cooldown=0,
min_lr=0)
tensorboard = CustomTensorBoard(
log_dir=tensorboard_logs,
write_graph=True,
write_images=True,
)
valid_map = EvaluationCallback(infer_model=model_to_save,
valid_generator=valid_generator,
labels=labels)
return [early_stop, checkpoint, reduce_on_plateau, tensorboard, valid_map]
def create_callbacks(model_log, tensorboard_logs, model_to_save):
makedirs(model_log)
early_stop = EarlyStopping(
monitor = 'loss',
min_delta = 0.01,
patience = 5,
mode = 'min',
verbose = 1
)
checkpoint = CustomModelCheckpoint(
model_to_save = model_to_save,
filepath = model_log + 'ep{epoch:03d}-loss{loss:.3f}.h5',
monitor = 'loss',
verbose = 1,
save_best_only = True,
mode = 'min',
period = 1
)
reduce_on_plateau = ReduceLROnPlateau(
monitor = 'loss',
factor = 0.1,
patience = 2,
verbose = 1,
mode = 'min',
epsilon = 0.01,
cooldown = 0,
min_lr = 0
)
tensorboard = CustomTensorBoard(
log_dir = model_log,
write_graph = True,
write_images = True,
)
return [early_stop, checkpoint, reduce_on_plateau, tensorboard]
def create_callbacks(saved_weights_name, model_to_save):
early_stop = EarlyStopping(monitor='val_loss',
min_delta=0.01,
patience=8,
mode='min',
verbose=1)
checkpoint = CustomModelCheckpoint(
model_to_save=model_to_save,
filepath=saved_weights_name, # + '{epoch:02d}.h5',
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min',
period=1)
reduce_on_plateau = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=3,
verbose=1,
mode='min',
epsilon=0.01,
cooldown=0,
min_lr=0)
return [early_stop, checkpoint, reduce_on_plateau]
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save):
makedirs(tensorboard_logs)
early_stop = EarlyStopping(monitor='loss',
min_delta=0.01,
patience=25,
mode='min',
verbose=1)
checkpoint = CustomModelCheckpoint(
model_to_save=model_to_save,
filepath=saved_weights_name, # + '{epoch:02d}.h5',
monitor='loss',
verbose=1,
save_best_only=True,
mode='min',
save_freq=1)
reduce_on_plateau = ReduceLROnPlateau(monitor='loss',
factor=0.5,
patience=15,
verbose=1,
mode='min',
min_delta=0.01,
cooldown=0,
min_lr=0)
tensorboard = CustomTensorBoard(
log_dir=tensorboard_logs,
write_graph=True,
write_images=True,
)
return [early_stop, checkpoint, reduce_on_plateau, tensorboard]
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save):
makedirs(tensorboard_logs)
early_stop = EarlyStopping( # early stop
monitor='loss', #based on loss
min_delta=0.01, #if improvment less than 0.01 it's not a real improve
patience=5, #number of times loss didn't improve to stop (5 times try )
mode='min', #based on loss choice we want it min
verbose=1 #print epoch and extra info default is 0
)
checkpoint = CustomModelCheckpoint(
model_to_save=model_to_save,
filepath=saved_weights_name, # + '{epoch:02d}.h5',
monitor='loss',
verbose=1,
save_best_only=True,
mode='min',
period=1)
reduce_on_plateau = ReduceLROnPlateau( # reduce learning rate when loss didn't improve
monitor='loss', #based ob loss
factor=0.1, #factir if reduce
patience=2, # every 2 epochs
verbose=1, # 0 equal quit , 1 update
mode='min', #min mode for loss
epsilon=0.01, #change in epsilon
cooldown=
0, #number of epochs to wait before resuming normal operation after lr has been reduced.
min_lr=0 #lower bound on the learning rate.
)
tensorboard = CustomTensorBoard(
log_dir=tensorboard_logs,
write_graph=True,
write_images=True,
)
return [early_stop, checkpoint, reduce_on_plateau, tensorboard]
def create_callbacks(config, model_to_save):
tensorboard_logs = config['train']['tensorboard_dir']
makedirs(tensorboard_logs)
early_stop = EarlyStopping(
monitor='loss',
min_delta=0.01,
patience=config['train']['early_stop']['patience'],
mode='min',
verbose=1)
checkpoint = CustomModelCheckpoint(
model_to_save=model_to_save,
filepath=config['train']['saved_weights_name'], # + '{epoch:02d}.h5',
monitor='loss',
verbose=1,
save_best_only=True,
mode='min',
period=1)
reduce_on_plateau = ReduceLROnPlateau(
monitor='loss',
factor=0.1,
patience=config['train']['reduce_on_plateau']['patience'],
verbose=1,
mode='min',
epsilon=0.01,
cooldown=0,
min_lr=0)
tensorboard = CustomTensorBoard(
log_dir=tensorboard_logs,
write_graph=True,
write_images=True,
)
return [early_stop, checkpoint, reduce_on_plateau, tensorboard]
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save):
makedirs(tensorboard_logs)
early_stop = EarlyStopping(monitor='loss',
min_delta=0.01,
patience=5,
mode='min',
verbose=1)
checkpoint = CustomModelCheckpoint(
model_to_save=model_to_save,
filepath=saved_weights_name, # + '{epoch:02d}.h5',
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min',
period=1)
reduce_on_plateau = ReduceLROnPlateau(monitor='loss',
factor=0.1,
patience=2,
verbose=1,
mode='min',
epsilon=0.01,
cooldown=0,
min_lr=0)
logdir = tensorboard_logs + os.path.split(
saved_weights_name)[-1][:-3] + ' ' + str(datetime.now())
tensorboard = CustomTensorBoard(
log_dir=logdir,
write_graph=True,
write_images=True,
)
return [checkpoint, reduce_on_plateau, tensorboard]
def get_callbacks(args):
callbacks_list = []
model_name = get_model_name(args)
# Save model weights
checkpoint = CustomModelCheckpoint(save_dir=os.path.join(
args.checkpoint_dir, model_name),
save_always=True)
callbacks_list.append(checkpoint)
# Save image samples
generate_samples = GenerateSamples(
examples_dir=os.path.join(args.examples_dir),
save_dir=os.path.join(args.sample_dir, model_name),
img_size=[args.img_w, args.img_h, args.img_ch])
callbacks_list.append(generate_samples)
if (args.architecture == 'munit'):
def scheduler(epoch, lr):
return lr * pow(0.5, epoch)
lr_scheduler = CustomLearningRateSchedule(scheduler,
init_gen_lr=args.gen_lr,
init_disc_lr=args.disc_lr,
verbose=1)
callbacks_list.append(lr_scheduler)
return callbacks_list
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save):
makedirs(tensorboard_logs)
early_stop = EarlyStopping(monitor='loss',
min_delta=0.01,
patience=7,
mode='min',
verbose=1)
checkpoint = CustomModelCheckpoint(
model_to_save=model_to_save,
filepath=saved_weights_name, # + '{epoch:02d}.h5',
monitor='loss',
verbose=1,
save_best_only=True,
mode='min',
period=1)
reduce_on_plateau = ReduceLROnPlateau(monitor='loss',
factor=0.1,
patience=2,
verbose=1,
mode='min',
epsilon=0.01,
cooldown=0,
min_lr=0)
tensorboard = CustomTensorBoard(
log_dir=tensorboard_logs,
write_graph=True,
write_images=True,
)
# Learning rate reduce
reduce_on_plateau = ReduceLROnPlateau(
monitor="loss", # val_accuracy, loss
factor=0.1,
patience=7,
verbose=1,
mode="min",
cooldown=0,
min_lr=0,
min_delta=0.01,
)
# Tensorboard log callback
tb_cb = tf.keras.callbacks.TensorBoard(log_dir=tensorboard_logs,
histogram_freq=0,
write_graph=True,
profile_batch=0)
# Model checkpoint callback
checkpoint_cb = ModelCheckpoint(saved_weights_name,
monitor="loss",
verbose=1,
save_best_only=True,
save_weights_only=True,
mode="min",
period=1)
return [reduce_on_plateau, checkpoint_cb]
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save):
makedirs(tensorboard_logs)
early_stop = EarlyStopping(monitor='loss',
min_delta=0.01,
patience=5,
mode='min',
verbose=1)
checkpoint = CustomModelCheckpoint(
model_to_save=model_to_save,
filepath=str(
os.path.join(tensorboard_logs,
saved_weights_name + '-{epoch:02d}.h5')),
monitor='loss',
verbose=1,
save_best_only=True,
mode='min',
period=1)
reduce_on_plateau = ReduceLROnPlateau(monitor='loss',
factor=0.1,
patience=3,
verbose=1,
mode='min',
epsilon=0.01,
cooldown=0,
min_lr=0)
tensorboard = CustomTensorBoard(
log_dir=tensorboard_logs,
write_graph=True,
write_images=True,
)
backup_checkpoint = CustomModelCheckpoint(
model_to_save=model_to_save,
filepath=str(
os.path.join(tensorboard_logs, saved_weights_name + '-backup.h5')),
monitor='loss',
verbose=1,
save_best_only=True,
mode='min',
period=1)
return [
early_stop, checkpoint, reduce_on_plateau, tensorboard,
backup_checkpoint
]
def get_callbacks():
callbacks = []
callbacks.append(TerminateOnNaN())
callbacks.append(ResultKeeper("results.bin"))
callbacks.append(CustomModelCheckpoint('checkpoints'))
callbacks.append(CustomTensorBoard('tensorboard'))
callbacks.append(
CustomEarlyStopping(mini_targets={
5: 200,
10: 100
},
monitor="val_loss",
patience=3))
#lr_decay = lambda epoch, lr: lr / np.power(.1, epoch)
#callbacks.append(LearningRateScheduler(lr_decay, verbose= 1))
return callbacks
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save):
makedirs(tensorboard_logs)
early_stop = EarlyStopping(monitor='loss',
min_delta=0.01,
patience=5,
mode='min',
verbose=1)
checkpoint = CustomModelCheckpoint(
model_to_save=model_to_save,
filepath=
'log_voc/ep{epoch:03d}-loss{loss:.3f}.h5', #saved_weights_name,# + '{epoch:02d}.h5',
monitor='loss',
verbose=1,
save_best_only=True,
mode='min',
period=1)
checkpoint2 = CustomModelCheckpoint2(
model_to_save=model_to_save,
filepath=saved_weights_name, # + '{epoch:02d}.h5',
monitor='loss',
verbose=1,
save_best_only=True,
mode='min',
period=1)
checkpoint3 = ModelCheckpoint('log_voc/ep{epoch:03d}-loss{loss:.3f}.h5',
monitor='loss',
save_best_only=True,
period=4)
#checkpoint3 = ModelCheckpoint('log_voc/ep{epoch:03d}-loss{loss:.3f}.h5', monitor='loss', save_weights_only=True, save_best_only=True, period=4)
#checkpoint2 not working yet for some reason... 1/15/2020
reduce_on_plateau = ReduceLROnPlateau(monitor='loss',
factor=0.1,
patience=2,
verbose=1,
mode='min',
epsilon=0.01,
cooldown=0,
min_lr=0)
tensorboard = CustomTensorBoard(
log_dir=tensorboard_logs,
write_graph=True,
write_images=True,
)
return [checkpoint, checkpoint2, tensorboard]
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save):
makedirs(tensorboard_logs)
logging.debug("Adding callback (Early Stopping)")
early_stop = EarlyStopping(monitor='val_loss',
min_delta=0.1,
patience=8,
mode='min',
verbose=1,
restore_best_weights=True)
logging.debug("Adding callback (CustomModelCheckpoint)")
checkpoint = CustomModelCheckpoint(
model_to_save=model_to_save,
filepath=saved_weights_name, # + '{epoch:02d}.h5',
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min',
period=1)
logging.debug("Adding callback (ReduceLROnPlateau)")
reduce_on_plateau = ReduceLROnPlateau(monitor='val_loss',
factor=0.2,
patience=5,
verbose=1,
mode='min',
epsilon=0.01,
cooldown=3,
min_lr=0.0)
logging.debug("Adding callback (CustomTensorBoard)")
tensorboard = CustomTensorBoard(
log_dir=tensorboard_logs,
write_graph=True,
write_images=True,
)
logging.debug("Adding callback (TerminateOnNaN)")
nan = tf.keras.callbacks.TerminateOnNaN()
logging.debug("Adding callback (LRTensorBoard)")
lr = LRTensorBoard(log_dir=tensorboard_logs)
return [early_stop, checkpoint, reduce_on_plateau, tensorboard, nan, lr]
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save,
textedit, cure, loss):
makedirs(tensorboard_logs)
early_stop = EarlyStopping(monitor='loss',
min_delta=0.01,
patience=5,
mode='min',
verbose=1)
checkpoint = CustomModelCheckpoint(
textedit,
cure,
loss,
model_to_save=model_to_save,
filepath='YOLO3_{epoch:02d}.h5', #saved_weights_name +,
monitor='loss',
verbose=1,
#save_best_only = True,
save_weights_only=False,
mode='min',
period=5)
reduce_on_plateau = ReduceLROnPlateau(monitor='loss',
factor=0.5,
patience=8,
verbose=1,
mode='min',
epsilon=0.01,
cooldown=0,
min_lr=0)
tensorboard = CustomTensorBoard(
log_dir=tensorboard_logs,
write_graph=True,
write_images=True,
)
return [ #early_stop,
checkpoint,
#reduce_on_plateau,
tensorboard
]
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save):
makedirs(tensorboard_logs)
yolo_create_exit('pass') \
if os.path.isdir(tensorboard_logs) \
else yolo_create_exit('fail')
early_stop = EarlyStopping(
monitor='loss',
min_delta=0.01,
patience=7,
mode='min',
verbose=1
)
checkpoint = CustomModelCheckpoint(
model_to_save=model_to_save,
filepath=saved_weights_name, # + '{epoch:02d}.h5',
monitor='loss',
verbose=1,
save_best_only=True,
mode='min',
period=1
)
reduce_on_plateau = ReduceLROnPlateau(
monitor='loss',
factor=0.1,
patience=2,
verbose=1,
mode='min',
epsilon=0.01,
cooldown=0,
min_lr=0
)
tensorboard = CustomTensorBoard(
log_dir=tensorboard_logs,
write_graph=True,
write_images=True,
)
return [early_stop, checkpoint, reduce_on_plateau, tensorboard]
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save):
makedirs(tensorboard_logs)
early_stop = EarlyStopping(monitor='loss',
min_delta=0.01,
patience=5,
mode='min',
verbose=1)
checkpoint = CustomModelCheckpoint(
model_to_save=model_to_save,
filepath=saved_weights_name, # + '{epoch:02d}.h5',
monitor='loss',
verbose=1,
save_best_only=True,
mode='min',
period=1)
tensorboard = CustomTensorBoard(
log_dir=tensorboard_logs,
write_graph=True,
write_images=True,
)
reduce_on_plateau = ReduceLROnPlateau(monitor='loss',
factor=0.1,
patience=2,
verbose=1,
mode='min',
epsilon=0.01,
cooldown=0,
min_lr=0)
lrate = LearningRateScheduler(get_current_learning_rate)
if WARMUP_EPOCHS <= 0:
return [early_stop, checkpoint, reduce_on_plateau, tensorboard]
return [lrate, early_stop, checkpoint, tensorboard]
def _main():
annotation_path = '2007_train.txt'
log_dir = 'logs/000/'
classes_path = 'model_data/voc_classes.txt'
anchors_path = 'model_data/yolo_anchors.txt'
class_names = get_classes(classes_path)
num_classes = len(class_names)
anchors = get_anchors(anchors_path)
print(anchors)
input_shape = (416,416) # multiple of 32, hw
print('flag.weights_file ', FLAGS.weights_file)
print('flag.skip_phasei ', FLAGS.skip_phasei)
#return
is_tiny_version = len(anchors)==6 # default setting
if is_tiny_version:
model = create_tiny_model(input_shape, anchors, num_classes,
freeze_body=2, weights_path='model_data/tiny_yolo_weights.h5')
else:
if FLAGS.weights_file=='':
model, model_to_save = create_model(input_shape, anchors, num_classes,
freeze_body=2, weights_path='model_data/darknet53_weights.h5') # make sure you know what you freeze
else:
model, model_to_save = create_model(input_shape, anchors, num_classes,
freeze_body=2, weights_path=FLAGS.weights_file) # continue train from a provided h5 file
logging = TensorBoard(log_dir=log_dir)
checkpoint = ModelCheckpoint(log_dir + 'ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}.h5',
monitor='val_loss', save_best_only=True, period=1)
checkpoint2 = CustomModelCheckpoint(
model_to_save = model_to_save,
filepath = log_dir+ 'epp{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}.h5', #saved_weights_name,# + '{epoch:02d}.h5',
monitor = 'val_loss',
verbose = 1,
save_best_only = True,
mode = 'min',
period = 4
)
#checkpoint = ModelCheckpoint(log_dir + 'ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}.h5',
# monitor='val_loss', save_weights_only=True, save_best_only=True, period=15)
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, verbose=1)
early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=10, verbose=1)
val_split = 0.1
with open(annotation_path) as f:
lines = f.readlines()
np.random.seed(10101)
np.random.shuffle(lines)
np.random.seed(None)
num_val = int(len(lines)*val_split)
num_train = len(lines) - num_val
# Train with frozen layers first, to get a stable loss.
# Adjust num epochs to your dataset. This step is enough to obtain a not bad model.
if not FLAGS.skip_phasei:
model.compile(optimizer=SGD(lr=1e-4), loss={
# use custom yolo_loss Lambda layer.
'yolo_loss': lambda y_true, y_pred: y_pred})
batch_size = 32
print('Train on {} samples, val on {} samples, with batch size {}.'.format(num_train, num_val, batch_size))
model.fit_generator(data_generator_wrapper(lines[:num_train], batch_size, input_shape, anchors, num_classes),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(lines[num_train:], batch_size, input_shape, anchors, num_classes),
validation_steps=max(1, num_val//batch_size),
epochs=FLAGS.phase_i,
initial_epoch=0,
callbacks=[logging, checkpoint2])
model_to_save.save(log_dir + 'trained_weights_stage_1.h5')
else:
print("\n\n---------------skipping phase i\n\n")
# Unfreeze and continue training, to fine-tune.
# Train longer if the result is not good.
if True:
for i in range(len(model.layers)):
model.layers[i].trainable = True
model.compile(optimizer=SGD(lr=1e-4), loss={'yolo_loss': lambda y_true, y_pred: y_pred}) # recompile to apply the change
print('Unfreeze all of the layers.')
batch_size = 4 # note that more GPU memory is required after unfreezing the body
print('Train on {} samples, val on {} samples, with batch size {}.'.format(num_train, num_val, batch_size))
model.fit_generator(data_generator_wrapper(lines[:num_train], batch_size, input_shape, anchors, num_classes),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(lines[num_train:], batch_size, input_shape, anchors, num_classes),
validation_steps=max(1, num_val//batch_size),
epochs=1100,
initial_epoch=1021,
callbacks=[logging, checkpoint2])
#callbacks=[logging, checkpoint, reduce_lr, early_stopping])
model_to_save.save(log_dir + 'trained_weights_final.h5')
def main():
config_path = args.conf
initial_weights = args.weights
with open(config_path) as config_buffer:
config = json.loads(config_buffer.read())
train_set, valid_set = data.create_training_instances(config['train']['train_folder'],
config['train']['train_masks'],
config['valid']['valid_folder'],
config['valid']['valid_masks'],
config['train']['cache_name'])
train_generator = gen.BatchGenerator(
instances = train_set,
batch_size = config['train']['batch_size'],
input_sz = config['model']['input_shape'],
shuffle = True,
jitter = 0.3,
norm = data.normalize,
downsample = 2
)
valid_generator = gen.BatchGenerator(
instances = valid_set,
batch_size = config['train']['batch_size'],
input_sz = config['model']['input_shape'],
norm = data.normalize,
infer = True,
downsample = 2
)
early_stop = EarlyStopping(
monitor = 'val_loss',
min_delta = 0,
patience = 100,
mode = 'min',
verbose = 1
)
reduce_on_plateau = ReduceLROnPlateau(
monitor = 'loss',
factor = 0.5,
patience = 5,
verbose = 1,
mode = 'min',
min_delta= 0.01,
cooldown = 0,
min_lr = 0
)
# Swapped as net input -> [H x W x C]
net_input_shape = (config['model']['input_shape'][1],
config['model']['input_shape'][0],
3)
train_model = models.create(
base = config['model']['base'],
input_shape = net_input_shape)
if initial_weights:
train_model.load_weights(initial_weights)
model_render_file = 'images/{}.png'.format(config['model']['base'])
if not os.path.isdir(os.path.dirname(model_render_file)):
os.makedirs(os.path.dirname(model_render_file))
plot_model(train_model, to_file=model_render_file, show_shapes=True)
# print_summary(train_model)
optimizer = Adam(lr=config['train']['learning_rate'], clipnorm=0.001)
# optimizer = SGD(lr=config['train']['learning_rate'], clipnorm=0.001)
train_model.compile(loss=models.result_loss, optimizer=optimizer,
metrics=[models.iou_loss, models.dice_coef_loss, models.pixelwise_crossentropy])
chk_name = config['train']['saved_weights_name']
chk_root, chk_ext = os.path.splitext(chk_name)
checkpoint_vloss = CustomModelCheckpoint(
model_to_save = train_model,
filepath = chk_root+'_ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}'+chk_ext,
monitor = 'val_loss',
verbose = 1,
save_best_only = True,
mode = 'min',
period = 1
)
if chk_name:
if not os.path.isdir(os.path.dirname(chk_name)):
os.makedirs(os.path.dirname(chk_name))
callbacks = [early_stop, reduce_on_plateau, checkpoint_vloss]
hist = train_model.fit_generator(
generator=train_generator,
steps_per_epoch=len(train_generator) * config['train']['train_times'],
validation_data=valid_generator,
validation_steps=len(valid_generator) * config['valid']['valid_times'],
epochs=config['train']['nb_epochs'],
verbose=2 if config['train']['debug'] else 1,
callbacks=callbacks,
workers=os.cpu_count(),
max_queue_size=100
)
if not os.path.exists('model'):
os.makedirs('model')
df = pd.DataFrame.from_dict(hist.history)
df.to_csv('model/hist.csv', encoding='utf-8', index=False)