def create_callbacks(training_model, prediction_model, validation_generator, args): """ Creates the callbacks to use during training. Args training_model: The model that is used for training. prediction_model: The model that should be used for validation. validation_generator: The generator for creating validation data. args: parseargs args object. Returns: A list of callbacks used for training. """ callbacks = [] tensorboard_callback = None # in tfv2 the way for logging custom scalars was changed # https://www.tensorflow.org/tensorboard/scalars_and_keras if args.tensorboard_dir: file_writer = tf.summary.create_file_writer(args.tensorboard_dir + "/metrics") file_writer.set_as_default() tensorboard_callback = tf.keras.callbacks.TensorBoard( log_dir=args.tensorboard_dir, histogram_freq=0, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) callbacks.append(tensorboard_callback) if args.evaluation and validation_generator: if args.dataset_type == 'coco': from eval.coco import CocoEval # use prediction model for evaluation evaluation = CocoEval(validation_generator, prediction_model, tensorboard=tensorboard_callback) else: from eval.pascal import Evaluate evaluation = Evaluate(validation_generator, prediction_model, tensorboard=tensorboard_callback) callbacks.append(evaluation) # save the model if args.snapshots: # ensure directory created first; otherwise h5py will error after epoch. makedirs(args.snapshot_path) checkpoint = tf.keras.callbacks.ModelCheckpoint( os.path.join( args.snapshot_path, '{dataset_type}_{{epoch:02d}}_{{loss:.4f}}_{{val_loss:.4f}}.h5' .format(dataset_type=args.dataset_type)), verbose=1, # save_best_only=True, # monitor="mAP", # mode='max' ) callbacks.append(checkpoint) # callbacks.append(keras.callbacks.ReduceLROnPlateau( # monitor='loss', # factor=0.1, # patience=2, # verbose=1, # mode='auto', # min_delta=0.0001, # cooldown=0, # min_lr=0 # )) return callbacks
def create_callbacks(training_model, prediction_model, validation_generator, args): """ Creates the callbacks to use during training. Args training_model: The model that is used for training. prediction_model: The model that should be used for validation. validation_generator: The generator for creating validation data. args: parseargs args object. Returns: A list of callbacks used for training. """ callbacks = [] tensorboard_callback = None if args.tensorboard_dir: tensorboard_callback = keras.callbacks.TensorBoard( log_dir=args.tensorboard_dir, histogram_freq=0, batch_size=args.batch_size, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) callbacks.append(tensorboard_callback) if args.evaluation and validation_generator: if args.dataset_type == 'coco': from eval.coco import CocoEval # use prediction model for evaluation evaluation = CocoEval(validation_generator, prediction_model, tensorboard=tensorboard_callback) else: from eval.pascal import Evaluate evaluation = Evaluate(validation_generator, prediction_model, tensorboard=tensorboard_callback) callbacks.append(evaluation) # save the model if args.snapshots: # ensure directory created first; otherwise h5py will error after epoch. makedirs(args.snapshot_path) checkpoint = keras.callbacks.ModelCheckpoint( os.path.join( args.snapshot_path, '{dataset_type}.h5'.format(dataset_type=args.dataset_type)), verbose=1, save_weights_only=True # save_best_only=True, # monitor="mAP", # mode='max' ) callbacks.append(checkpoint) # callbacks.append(keras.callbacks.ReduceLROnPlateau( # monitor='loss', # factor=0.1, # patience=2, # verbose=1, # mode='auto', # min_delta=0.0001, # cooldown=0, # min_lr=0 # )) return callbacks