def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save, valid_generator, config): early_stop = EarlyStopping( monitor = 'loss', min_delta = 0.01, patience = 5, mode = 'min', verbose = 1 ) checkpoint = CustomModelCheckpoint( model_to_save = model_to_save, filepath = saved_weights_name + '-epoch_{epoch:02d}-val_loss_{val_loss:.04f}.h5', monitor = 'val_loss', verbose = 1, save_best_only = True, mode = 'min', period = 1 ) my_checkpoint = My_Checkpoint( model_to_save=model_to_save, valid_generator = valid_generator, config = config ) only_one_checkpoint = CustomModelCheckpoint( model_to_save=model_to_save, filepath=saved_weights_name + '.h5', monitor='val_loss', verbose=1, save_best_only=True, mode='min', period=1 ) reduce_on_plateau = ReduceLROnPlateau( monitor = 'loss', factor = 0.1, patience = 2, verbose = 1, mode = 'min', epsilon = 0.01, cooldown = 0, min_lr = 0 ) # makedirs(tensorboard_logs) # tensorboard = CustomTensorBoard( # log_dir = tensorboard_logs, # write_graph = True, # write_images = True, # ) return [early_stop, checkpoint, only_one_checkpoint, reduce_on_plateau, my_checkpoint]
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save): makedirs(tensorboard_logs) early_stop = EarlyStopping(monitor='loss', min_delta=0.01, patience=5, mode='min', verbose=1) checkpoint = CustomModelCheckpoint( model_to_save=model_to_save, filepath=saved_weights_name, # + '{epoch:02d}.h5', monitor='loss', verbose=1, save_best_only=True, mode='min', period=1) checkpoint2 = CustomModelCheckpoint( model_to_save=model_to_save, filepath="watermark_val.h5", # + '{epoch:02d}.h5', monitor='val_loss', verbose=1, save_best_only=True, mode='auto', period=1) checkpoint3 = CustomModelCheckpoint( model_to_save=model_to_save, filepath="watermark_valacc.h5", # + '{epoch:02d}.h5', monitor='val_acc', verbose=1, save_best_only=True, mode='auto', period=1) reduce_on_plateau = ReduceLROnPlateau(monitor='loss', factor=0.1, patience=2, verbose=1, mode='min', epsilon=0.01, cooldown=0, min_lr=0) tensorboard = CustomTensorBoard( log_dir=tensorboard_logs, write_graph=True, write_images=True, ) return [ early_stop, checkpoint, checkpoint2, checkpoint3, reduce_on_plateau, tensorboard ]
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save, valid_generator, labels): makedirs(tensorboard_logs) early_stop = EarlyStopping(monitor='val_loss', min_delta=0.01, patience=7, mode='min', verbose=1) checkpoint = CustomModelCheckpoint( model_to_save=model_to_save, filepath=saved_weights_name, # + '{epoch:02d}.h5', monitor='val_loss', verbose=1, save_best_only=True, mode='min', period=1) reduce_on_plateau = ReduceLROnPlateau(monitor='loss', factor=0.1, patience=2, verbose=1, mode='min', epsilon=0.01, cooldown=0, min_lr=0) tensorboard = CustomTensorBoard( log_dir=tensorboard_logs, write_graph=True, write_images=True, ) valid_map = EvaluationCallback(infer_model=model_to_save, valid_generator=valid_generator, labels=labels) return [early_stop, checkpoint, reduce_on_plateau, tensorboard, valid_map]
def create_callbacks(model_log, tensorboard_logs, model_to_save): makedirs(model_log) early_stop = EarlyStopping( monitor = 'loss', min_delta = 0.01, patience = 5, mode = 'min', verbose = 1 ) checkpoint = CustomModelCheckpoint( model_to_save = model_to_save, filepath = model_log + 'ep{epoch:03d}-loss{loss:.3f}.h5', monitor = 'loss', verbose = 1, save_best_only = True, mode = 'min', period = 1 ) reduce_on_plateau = ReduceLROnPlateau( monitor = 'loss', factor = 0.1, patience = 2, verbose = 1, mode = 'min', epsilon = 0.01, cooldown = 0, min_lr = 0 ) tensorboard = CustomTensorBoard( log_dir = model_log, write_graph = True, write_images = True, ) return [early_stop, checkpoint, reduce_on_plateau, tensorboard]
def create_callbacks(saved_weights_name, model_to_save): early_stop = EarlyStopping(monitor='val_loss', min_delta=0.01, patience=8, mode='min', verbose=1) checkpoint = CustomModelCheckpoint( model_to_save=model_to_save, filepath=saved_weights_name, # + '{epoch:02d}.h5', monitor='val_loss', verbose=1, save_best_only=True, mode='min', period=1) reduce_on_plateau = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, verbose=1, mode='min', epsilon=0.01, cooldown=0, min_lr=0) return [early_stop, checkpoint, reduce_on_plateau]
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save): makedirs(tensorboard_logs) early_stop = EarlyStopping(monitor='loss', min_delta=0.01, patience=25, mode='min', verbose=1) checkpoint = CustomModelCheckpoint( model_to_save=model_to_save, filepath=saved_weights_name, # + '{epoch:02d}.h5', monitor='loss', verbose=1, save_best_only=True, mode='min', save_freq=1) reduce_on_plateau = ReduceLROnPlateau(monitor='loss', factor=0.5, patience=15, verbose=1, mode='min', min_delta=0.01, cooldown=0, min_lr=0) tensorboard = CustomTensorBoard( log_dir=tensorboard_logs, write_graph=True, write_images=True, ) return [early_stop, checkpoint, reduce_on_plateau, tensorboard]
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save): makedirs(tensorboard_logs) early_stop = EarlyStopping( # early stop monitor='loss', #based on loss min_delta=0.01, #if improvment less than 0.01 it's not a real improve patience=5, #number of times loss didn't improve to stop (5 times try ) mode='min', #based on loss choice we want it min verbose=1 #print epoch and extra info default is 0 ) checkpoint = CustomModelCheckpoint( model_to_save=model_to_save, filepath=saved_weights_name, # + '{epoch:02d}.h5', monitor='loss', verbose=1, save_best_only=True, mode='min', period=1) reduce_on_plateau = ReduceLROnPlateau( # reduce learning rate when loss didn't improve monitor='loss', #based ob loss factor=0.1, #factir if reduce patience=2, # every 2 epochs verbose=1, # 0 equal quit , 1 update mode='min', #min mode for loss epsilon=0.01, #change in epsilon cooldown= 0, #number of epochs to wait before resuming normal operation after lr has been reduced. min_lr=0 #lower bound on the learning rate. ) tensorboard = CustomTensorBoard( log_dir=tensorboard_logs, write_graph=True, write_images=True, ) return [early_stop, checkpoint, reduce_on_plateau, tensorboard]
def create_callbacks(config, model_to_save): tensorboard_logs = config['train']['tensorboard_dir'] makedirs(tensorboard_logs) early_stop = EarlyStopping( monitor='loss', min_delta=0.01, patience=config['train']['early_stop']['patience'], mode='min', verbose=1) checkpoint = CustomModelCheckpoint( model_to_save=model_to_save, filepath=config['train']['saved_weights_name'], # + '{epoch:02d}.h5', monitor='loss', verbose=1, save_best_only=True, mode='min', period=1) reduce_on_plateau = ReduceLROnPlateau( monitor='loss', factor=0.1, patience=config['train']['reduce_on_plateau']['patience'], verbose=1, mode='min', epsilon=0.01, cooldown=0, min_lr=0) tensorboard = CustomTensorBoard( log_dir=tensorboard_logs, write_graph=True, write_images=True, ) return [early_stop, checkpoint, reduce_on_plateau, tensorboard]
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save): makedirs(tensorboard_logs) early_stop = EarlyStopping(monitor='loss', min_delta=0.01, patience=5, mode='min', verbose=1) checkpoint = CustomModelCheckpoint( model_to_save=model_to_save, filepath=saved_weights_name, # + '{epoch:02d}.h5', monitor='val_loss', verbose=1, save_best_only=True, mode='min', period=1) reduce_on_plateau = ReduceLROnPlateau(monitor='loss', factor=0.1, patience=2, verbose=1, mode='min', epsilon=0.01, cooldown=0, min_lr=0) logdir = tensorboard_logs + os.path.split( saved_weights_name)[-1][:-3] + ' ' + str(datetime.now()) tensorboard = CustomTensorBoard( log_dir=logdir, write_graph=True, write_images=True, ) return [checkpoint, reduce_on_plateau, tensorboard]
def get_callbacks(args): callbacks_list = [] model_name = get_model_name(args) # Save model weights checkpoint = CustomModelCheckpoint(save_dir=os.path.join( args.checkpoint_dir, model_name), save_always=True) callbacks_list.append(checkpoint) # Save image samples generate_samples = GenerateSamples( examples_dir=os.path.join(args.examples_dir), save_dir=os.path.join(args.sample_dir, model_name), img_size=[args.img_w, args.img_h, args.img_ch]) callbacks_list.append(generate_samples) if (args.architecture == 'munit'): def scheduler(epoch, lr): return lr * pow(0.5, epoch) lr_scheduler = CustomLearningRateSchedule(scheduler, init_gen_lr=args.gen_lr, init_disc_lr=args.disc_lr, verbose=1) callbacks_list.append(lr_scheduler) return callbacks_list
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save): makedirs(tensorboard_logs) early_stop = EarlyStopping(monitor='loss', min_delta=0.01, patience=7, mode='min', verbose=1) checkpoint = CustomModelCheckpoint( model_to_save=model_to_save, filepath=saved_weights_name, # + '{epoch:02d}.h5', monitor='loss', verbose=1, save_best_only=True, mode='min', period=1) reduce_on_plateau = ReduceLROnPlateau(monitor='loss', factor=0.1, patience=2, verbose=1, mode='min', epsilon=0.01, cooldown=0, min_lr=0) tensorboard = CustomTensorBoard( log_dir=tensorboard_logs, write_graph=True, write_images=True, ) # Learning rate reduce reduce_on_plateau = ReduceLROnPlateau( monitor="loss", # val_accuracy, loss factor=0.1, patience=7, verbose=1, mode="min", cooldown=0, min_lr=0, min_delta=0.01, ) # Tensorboard log callback tb_cb = tf.keras.callbacks.TensorBoard(log_dir=tensorboard_logs, histogram_freq=0, write_graph=True, profile_batch=0) # Model checkpoint callback checkpoint_cb = ModelCheckpoint(saved_weights_name, monitor="loss", verbose=1, save_best_only=True, save_weights_only=True, mode="min", period=1) return [reduce_on_plateau, checkpoint_cb]
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save): makedirs(tensorboard_logs) early_stop = EarlyStopping(monitor='loss', min_delta=0.01, patience=5, mode='min', verbose=1) checkpoint = CustomModelCheckpoint( model_to_save=model_to_save, filepath=str( os.path.join(tensorboard_logs, saved_weights_name + '-{epoch:02d}.h5')), monitor='loss', verbose=1, save_best_only=True, mode='min', period=1) reduce_on_plateau = ReduceLROnPlateau(monitor='loss', factor=0.1, patience=3, verbose=1, mode='min', epsilon=0.01, cooldown=0, min_lr=0) tensorboard = CustomTensorBoard( log_dir=tensorboard_logs, write_graph=True, write_images=True, ) backup_checkpoint = CustomModelCheckpoint( model_to_save=model_to_save, filepath=str( os.path.join(tensorboard_logs, saved_weights_name + '-backup.h5')), monitor='loss', verbose=1, save_best_only=True, mode='min', period=1) return [ early_stop, checkpoint, reduce_on_plateau, tensorboard, backup_checkpoint ]
def get_callbacks(): callbacks = [] callbacks.append(TerminateOnNaN()) callbacks.append(ResultKeeper("results.bin")) callbacks.append(CustomModelCheckpoint('checkpoints')) callbacks.append(CustomTensorBoard('tensorboard')) callbacks.append( CustomEarlyStopping(mini_targets={ 5: 200, 10: 100 }, monitor="val_loss", patience=3)) #lr_decay = lambda epoch, lr: lr / np.power(.1, epoch) #callbacks.append(LearningRateScheduler(lr_decay, verbose= 1)) return callbacks
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save): makedirs(tensorboard_logs) early_stop = EarlyStopping(monitor='loss', min_delta=0.01, patience=5, mode='min', verbose=1) checkpoint = CustomModelCheckpoint( model_to_save=model_to_save, filepath= 'log_voc/ep{epoch:03d}-loss{loss:.3f}.h5', #saved_weights_name,# + '{epoch:02d}.h5', monitor='loss', verbose=1, save_best_only=True, mode='min', period=1) checkpoint2 = CustomModelCheckpoint2( model_to_save=model_to_save, filepath=saved_weights_name, # + '{epoch:02d}.h5', monitor='loss', verbose=1, save_best_only=True, mode='min', period=1) checkpoint3 = ModelCheckpoint('log_voc/ep{epoch:03d}-loss{loss:.3f}.h5', monitor='loss', save_best_only=True, period=4) #checkpoint3 = ModelCheckpoint('log_voc/ep{epoch:03d}-loss{loss:.3f}.h5', monitor='loss', save_weights_only=True, save_best_only=True, period=4) #checkpoint2 not working yet for some reason... 1/15/2020 reduce_on_plateau = ReduceLROnPlateau(monitor='loss', factor=0.1, patience=2, verbose=1, mode='min', epsilon=0.01, cooldown=0, min_lr=0) tensorboard = CustomTensorBoard( log_dir=tensorboard_logs, write_graph=True, write_images=True, ) return [checkpoint, checkpoint2, tensorboard]
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save): makedirs(tensorboard_logs) logging.debug("Adding callback (Early Stopping)") early_stop = EarlyStopping(monitor='val_loss', min_delta=0.1, patience=8, mode='min', verbose=1, restore_best_weights=True) logging.debug("Adding callback (CustomModelCheckpoint)") checkpoint = CustomModelCheckpoint( model_to_save=model_to_save, filepath=saved_weights_name, # + '{epoch:02d}.h5', monitor='val_loss', verbose=1, save_best_only=True, mode='min', period=1) logging.debug("Adding callback (ReduceLROnPlateau)") reduce_on_plateau = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5, verbose=1, mode='min', epsilon=0.01, cooldown=3, min_lr=0.0) logging.debug("Adding callback (CustomTensorBoard)") tensorboard = CustomTensorBoard( log_dir=tensorboard_logs, write_graph=True, write_images=True, ) logging.debug("Adding callback (TerminateOnNaN)") nan = tf.keras.callbacks.TerminateOnNaN() logging.debug("Adding callback (LRTensorBoard)") lr = LRTensorBoard(log_dir=tensorboard_logs) return [early_stop, checkpoint, reduce_on_plateau, tensorboard, nan, lr]
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save, textedit, cure, loss): makedirs(tensorboard_logs) early_stop = EarlyStopping(monitor='loss', min_delta=0.01, patience=5, mode='min', verbose=1) checkpoint = CustomModelCheckpoint( textedit, cure, loss, model_to_save=model_to_save, filepath='YOLO3_{epoch:02d}.h5', #saved_weights_name +, monitor='loss', verbose=1, #save_best_only = True, save_weights_only=False, mode='min', period=5) reduce_on_plateau = ReduceLROnPlateau(monitor='loss', factor=0.5, patience=8, verbose=1, mode='min', epsilon=0.01, cooldown=0, min_lr=0) tensorboard = CustomTensorBoard( log_dir=tensorboard_logs, write_graph=True, write_images=True, ) return [ #early_stop, checkpoint, #reduce_on_plateau, tensorboard ]
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save): makedirs(tensorboard_logs) yolo_create_exit('pass') \ if os.path.isdir(tensorboard_logs) \ else yolo_create_exit('fail') early_stop = EarlyStopping( monitor='loss', min_delta=0.01, patience=7, mode='min', verbose=1 ) checkpoint = CustomModelCheckpoint( model_to_save=model_to_save, filepath=saved_weights_name, # + '{epoch:02d}.h5', monitor='loss', verbose=1, save_best_only=True, mode='min', period=1 ) reduce_on_plateau = ReduceLROnPlateau( monitor='loss', factor=0.1, patience=2, verbose=1, mode='min', epsilon=0.01, cooldown=0, min_lr=0 ) tensorboard = CustomTensorBoard( log_dir=tensorboard_logs, write_graph=True, write_images=True, ) return [early_stop, checkpoint, reduce_on_plateau, tensorboard]
def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save): makedirs(tensorboard_logs) early_stop = EarlyStopping(monitor='loss', min_delta=0.01, patience=5, mode='min', verbose=1) checkpoint = CustomModelCheckpoint( model_to_save=model_to_save, filepath=saved_weights_name, # + '{epoch:02d}.h5', monitor='loss', verbose=1, save_best_only=True, mode='min', period=1) tensorboard = CustomTensorBoard( log_dir=tensorboard_logs, write_graph=True, write_images=True, ) reduce_on_plateau = ReduceLROnPlateau(monitor='loss', factor=0.1, patience=2, verbose=1, mode='min', epsilon=0.01, cooldown=0, min_lr=0) lrate = LearningRateScheduler(get_current_learning_rate) if WARMUP_EPOCHS <= 0: return [early_stop, checkpoint, reduce_on_plateau, tensorboard] return [lrate, early_stop, checkpoint, tensorboard]
def _main(): annotation_path = '2007_train.txt' log_dir = 'logs/000/' classes_path = 'model_data/voc_classes.txt' anchors_path = 'model_data/yolo_anchors.txt' class_names = get_classes(classes_path) num_classes = len(class_names) anchors = get_anchors(anchors_path) print(anchors) input_shape = (416,416) # multiple of 32, hw print('flag.weights_file ', FLAGS.weights_file) print('flag.skip_phasei ', FLAGS.skip_phasei) #return is_tiny_version = len(anchors)==6 # default setting if is_tiny_version: model = create_tiny_model(input_shape, anchors, num_classes, freeze_body=2, weights_path='model_data/tiny_yolo_weights.h5') else: if FLAGS.weights_file=='': model, model_to_save = create_model(input_shape, anchors, num_classes, freeze_body=2, weights_path='model_data/darknet53_weights.h5') # make sure you know what you freeze else: model, model_to_save = create_model(input_shape, anchors, num_classes, freeze_body=2, weights_path=FLAGS.weights_file) # continue train from a provided h5 file logging = TensorBoard(log_dir=log_dir) checkpoint = ModelCheckpoint(log_dir + 'ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}.h5', monitor='val_loss', save_best_only=True, period=1) checkpoint2 = CustomModelCheckpoint( model_to_save = model_to_save, filepath = log_dir+ 'epp{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}.h5', #saved_weights_name,# + '{epoch:02d}.h5', monitor = 'val_loss', verbose = 1, save_best_only = True, mode = 'min', period = 4 ) #checkpoint = ModelCheckpoint(log_dir + 'ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}.h5', # monitor='val_loss', save_weights_only=True, save_best_only=True, period=15) reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, verbose=1) early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=10, verbose=1) val_split = 0.1 with open(annotation_path) as f: lines = f.readlines() np.random.seed(10101) np.random.shuffle(lines) np.random.seed(None) num_val = int(len(lines)*val_split) num_train = len(lines) - num_val # Train with frozen layers first, to get a stable loss. # Adjust num epochs to your dataset. This step is enough to obtain a not bad model. if not FLAGS.skip_phasei: model.compile(optimizer=SGD(lr=1e-4), loss={ # use custom yolo_loss Lambda layer. 'yolo_loss': lambda y_true, y_pred: y_pred}) batch_size = 32 print('Train on {} samples, val on {} samples, with batch size {}.'.format(num_train, num_val, batch_size)) model.fit_generator(data_generator_wrapper(lines[:num_train], batch_size, input_shape, anchors, num_classes), steps_per_epoch=max(1, num_train//batch_size), validation_data=data_generator_wrapper(lines[num_train:], batch_size, input_shape, anchors, num_classes), validation_steps=max(1, num_val//batch_size), epochs=FLAGS.phase_i, initial_epoch=0, callbacks=[logging, checkpoint2]) model_to_save.save(log_dir + 'trained_weights_stage_1.h5') else: print("\n\n---------------skipping phase i\n\n") # Unfreeze and continue training, to fine-tune. # Train longer if the result is not good. if True: for i in range(len(model.layers)): model.layers[i].trainable = True model.compile(optimizer=SGD(lr=1e-4), loss={'yolo_loss': lambda y_true, y_pred: y_pred}) # recompile to apply the change print('Unfreeze all of the layers.') batch_size = 4 # note that more GPU memory is required after unfreezing the body print('Train on {} samples, val on {} samples, with batch size {}.'.format(num_train, num_val, batch_size)) model.fit_generator(data_generator_wrapper(lines[:num_train], batch_size, input_shape, anchors, num_classes), steps_per_epoch=max(1, num_train//batch_size), validation_data=data_generator_wrapper(lines[num_train:], batch_size, input_shape, anchors, num_classes), validation_steps=max(1, num_val//batch_size), epochs=1100, initial_epoch=1021, callbacks=[logging, checkpoint2]) #callbacks=[logging, checkpoint, reduce_lr, early_stopping]) model_to_save.save(log_dir + 'trained_weights_final.h5')
def main(): config_path = args.conf initial_weights = args.weights with open(config_path) as config_buffer: config = json.loads(config_buffer.read()) train_set, valid_set = data.create_training_instances(config['train']['train_folder'], config['train']['train_masks'], config['valid']['valid_folder'], config['valid']['valid_masks'], config['train']['cache_name']) train_generator = gen.BatchGenerator( instances = train_set, batch_size = config['train']['batch_size'], input_sz = config['model']['input_shape'], shuffle = True, jitter = 0.3, norm = data.normalize, downsample = 2 ) valid_generator = gen.BatchGenerator( instances = valid_set, batch_size = config['train']['batch_size'], input_sz = config['model']['input_shape'], norm = data.normalize, infer = True, downsample = 2 ) early_stop = EarlyStopping( monitor = 'val_loss', min_delta = 0, patience = 100, mode = 'min', verbose = 1 ) reduce_on_plateau = ReduceLROnPlateau( monitor = 'loss', factor = 0.5, patience = 5, verbose = 1, mode = 'min', min_delta= 0.01, cooldown = 0, min_lr = 0 ) # Swapped as net input -> [H x W x C] net_input_shape = (config['model']['input_shape'][1], config['model']['input_shape'][0], 3) train_model = models.create( base = config['model']['base'], input_shape = net_input_shape) if initial_weights: train_model.load_weights(initial_weights) model_render_file = 'images/{}.png'.format(config['model']['base']) if not os.path.isdir(os.path.dirname(model_render_file)): os.makedirs(os.path.dirname(model_render_file)) plot_model(train_model, to_file=model_render_file, show_shapes=True) # print_summary(train_model) optimizer = Adam(lr=config['train']['learning_rate'], clipnorm=0.001) # optimizer = SGD(lr=config['train']['learning_rate'], clipnorm=0.001) train_model.compile(loss=models.result_loss, optimizer=optimizer, metrics=[models.iou_loss, models.dice_coef_loss, models.pixelwise_crossentropy]) chk_name = config['train']['saved_weights_name'] chk_root, chk_ext = os.path.splitext(chk_name) checkpoint_vloss = CustomModelCheckpoint( model_to_save = train_model, filepath = chk_root+'_ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}'+chk_ext, monitor = 'val_loss', verbose = 1, save_best_only = True, mode = 'min', period = 1 ) if chk_name: if not os.path.isdir(os.path.dirname(chk_name)): os.makedirs(os.path.dirname(chk_name)) callbacks = [early_stop, reduce_on_plateau, checkpoint_vloss] hist = train_model.fit_generator( generator=train_generator, steps_per_epoch=len(train_generator) * config['train']['train_times'], validation_data=valid_generator, validation_steps=len(valid_generator) * config['valid']['valid_times'], epochs=config['train']['nb_epochs'], verbose=2 if config['train']['debug'] else 1, callbacks=callbacks, workers=os.cpu_count(), max_queue_size=100 ) if not os.path.exists('model'): os.makedirs('model') df = pd.DataFrame.from_dict(hist.history) df.to_csv('model/hist.csv', encoding='utf-8', index=False)