def create_callbacks(model, training_model, prediction_model, validation_generator, dataset_type): callbacks = [] # save the prediction model checkpoint = keras.callbacks.ModelCheckpoint(os.path.join( 'snapshots', 'resnet50_{dataset_type}_{{epoch:02d}}.h5'.format( dataset_type=dataset_type)), verbose=1) checkpoint = RedirectModel(checkpoint, prediction_model) callbacks.append(checkpoint) if dataset_type == 'coco': import keras_retinanet.callbacks.coco # use prediction model for evaluation evaluation = keras_retinanet.callbacks.coco.CocoEval( validation_generator) evaluation = RedirectModel(evaluation, prediction_model) callbacks.append(evaluation) lr_scheduler = keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.1, patience=2, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0) callbacks.append(lr_scheduler) return callbacks
def create_callbacks(model, training_model, prediction_model, train_generator, validation_generator, p): callbacks = [] # save the prediction model if p['snapshots']: checkpoint = keras.callbacks.ModelCheckpoint( os.path.join( p['snapshot-path'], 'resnet'+str(p['resnet'])+'_{epoch:02d}.h5' ), verbose=1 ) checkpoint = RedirectModel(checkpoint, prediction_model) callbacks.append(checkpoint) # from ..callbacks.coco import CocoEval # evaluation = CocoEval(validation_generator) # callbacks.append(evaluation) if p['lr-plateau'] == True: lr_scheduler = keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.1, patience=2, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0) callbacks.append(lr_scheduler) if p['batch-log'] == True: global batch_log_file batch_log_file = open(p['batch-log-name'],'a') logging_callback = keras.callbacks.LambdaCallback( on_batch_end = lambda batch, logs: batch_log_file.write('{},{}\n'.format(batch,logs['loss'])) ) callbacks.append(logging_callback) if p['loss'] == True: global valid_loss_file global train_loss_file valid_loss_file = open(p['loss-folder-path']+'valid-loss.csv','a') train_loss_file = open(p['loss-folder-path']+'train-loss.csv','a') valid_loss_callback = keras.callbacks.LambdaCallback( on_epoch_end = lambda epoch, logs: valid_loss_file.write('{},{}\n'.format(epoch,logs['val_loss'])) ) train_loss_callback = keras.callbacks.LambdaCallback( on_epoch_end = lambda epoch, logs: train_loss_file.write('{},{}\n'.format(epoch,logs['loss'])) ) callbacks.append(valid_loss_callback) callbacks.append(train_loss_callback) if p['valid-acc'] == True: evaluation = LogoEval(validation_generator,filepath=p['valid-acc-path'],threshold=p['threshold'],iou_threshold=p['iou-threshold']) evaluation = RedirectModel(evaluation, prediction_model) callbacks.append(evaluation) if p['train-acc'] == True: evaluation = LogoEval(train_generator,filepath=p['train-acc-path'],threshold=p['threshold'],iou_threshold=p['iou-threshold']) evaluation = RedirectModel(evaluation, prediction_model) callbacks.append(evaluation) return callbacks
def create_callbacks(model, training_model, prediction_model, validation_generator, args, create_evaluation=Evaluate): callbacks = [] # save the prediction model if args.snapshots: # ensure directory created first; otherwise h5py will error after epoch. os.makedirs(args.snapshot_path, exist_ok=True) checkpoint = keras.callbacks.ModelCheckpoint( os.path.join( args.snapshot_path, '{backbone}_{dataset_type}_{{epoch:02d}}.h5'.format(backbone=args.backbone, dataset_type=args.dataset_type) ), verbose=1 ) checkpoint = RedirectModel(checkpoint, model) callbacks.append(checkpoint) tensorboard_callback = None if args.tensorboard_dir: tensorboard_callback = keras.callbacks.TensorBoard( log_dir = args.tensorboard_dir, histogram_freq = 0, batch_size = args.batch_size, write_graph = True, write_grads = False, write_images = False, embeddings_freq = 0, embeddings_layer_names = None, embeddings_metadata = None ) callbacks.append(tensorboard_callback) if args.evaluation and validation_generator: if args.dataset_type == 'coco': from ..callbacks.coco import CocoEval # use prediction model for evaluation evaluation = CocoEval(validation_generator) elif create_evaluation: evaluation = create_evaluation(validation_generator, tensorboard=tensorboard_callback, weighted_average=args.weighted_average) else: evaluation = Evaluate(validation_generator, tensorboard=tensorboard_callback, weighted_average=args.weighted_average) evaluation = RedirectModel(evaluation, prediction_model) callbacks.append(evaluation) callbacks.append(keras.callbacks.ReduceLROnPlateau( monitor = 'loss', factor = 0.1, patience = 2, verbose = 1, mode = 'auto', epsilon = 0.0001, cooldown = 0, min_lr = 0 )) return callbacks
def create_callbacks(model, training_model, prediction_model, validation_generator, args, comet_experiment): """Creates the callbacks to use during training. Args model: The base model. training_model: The model that is used for training. prediction_model: The model that should be used for validation. validation_generator: The generator for creating validation data. args: parseargs args object. comet_experiment: cometml object to log images Returns: A list of callbacks used for training. """ callbacks = [] if args.evaluation and validation_generator: evaluation = Evaluate(validation_generator, weighted_average=args.weighted_average, comet_experiment=comet_experiment, save_path=args.save_path, score_threshold=args.score_threshold) evaluation = RedirectModel(evaluation, prediction_model) callbacks.append(evaluation) # save the model if args.snapshots: # ensure directory created first; otherwise h5py will error after epoch. makedirs(args.snapshot_path) checkpoint = keras.callbacks.ModelCheckpoint( os.path.join( args.snapshot_path, '{backbone}_{dataset_type}_{{epoch:02d}}.h5'.format( backbone=args.backbone, dataset_type=args.dataset_type)), verbose=1, # save_best_only=True, # monitor="mAP", # mode='max' ) checkpoint = RedirectModel(checkpoint, model) callbacks.append(checkpoint) callbacks.append( keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.1, patience=2, verbose=1, mode='auto', min_delta=0.0001, cooldown=0, min_lr=0)) return callbacks
def create_callbacks(model, training_model, prediction_model, validation_generator, args): callbacks = [] # save the prediction model if args.snapshots: # ensure directory created first; otherwise h5py will error after epoch. io_utils.create_folder(args.snapshot_path) checkpoint = keras.callbacks.ModelCheckpoint(os.path.join( args.snapshot_path, '{backbone}_{dataset_type}_{{epoch:02d}}.h5'.format( backbone=args.backbone, dataset_type=args.dataset_type)), verbose=1) checkpoint = RedirectModel(checkpoint, prediction_model) callbacks.append(checkpoint) tensorboard_callback = None if args.tensorboard_dir: tensorboard_callback = keras.callbacks.TensorBoard( log_dir=args.tensorboard_dir, histogram_freq=0, batch_size=args.batch_size, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) callbacks.append(tensorboard_callback) if args.evaluation and validation_generator: if args.dataset_type == 'traffic_signs': # use prediction model for evaluation ground_truth_proto_file = os.path.join(args.val_path, 'rois.bin') evaluation = TrafficSignsEval( validation_generator, ground_truth_proto_file, os.path.join(args.train_path, 'rois.bin')) evaluation = RedirectModel(evaluation, prediction_model) callbacks.append(evaluation) callbacks.append( keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.1, patience=2, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0)) return callbacks
def create_callbacks(model, args): """ Creates the callbacks to use during training. Args model: The base model. training_model: The model that is used for training. prediction_model: The model that should be used for validation. validation_generator: The generator for creating validation data. args: parseargs args object. Returns: A list of callbacks used for training. """ callbacks = [] tensorboard_callback = None if args.tensorboard_dir: tensorboard_callback = keras.callbacks.TensorBoard( log_dir=args.tensorboard_dir, histogram_freq=0, batch_size=args.batch_size, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) callbacks.append(tensorboard_callback) # save the model if args.snapshots: # ensure directory created first; otherwise h5py will error after epoch. makedirs(args.snapshot_path) checkpoint = keras.callbacks.ModelCheckpoint( os.path.join( args.snapshot_path, '{backbone}_{dataset_type}_level_{level}_{{epoch:02d}}.h5'. format(backbone=args.backbone, level=args.label_level, dataset_type=args.dataset_type)), verbose=1, # save_best_only=True, # monitor="mAP", # mode='max' ) checkpoint = RedirectModel(checkpoint, model) callbacks.append(checkpoint) callbacks.append( keras.callbacks.ReduceLROnPlateau( monitor='loss', factor=0.8, patience=2, verbose=1, mode='auto', cooldown=0, min_lr=5e-6, )) return callbacks
def create_callbacks(model, training_model, prediction_model, validation_generator, dataset_type, snapshot_path, args): callbacks = [] # save the prediction model checkpoint = keras.callbacks.ModelCheckpoint(os.path.join( snapshot_path, '{backbone}_{dataset_type}_{{epoch:02d}}.h5'.format( backbone=args.backbone, dataset_type=dataset_type)), verbose=1) checkpoint = RedirectModel(checkpoint, prediction_model) callbacks.append(checkpoint) if args.log_dir: tensorboard_callback = keras.callbacks.TensorBoard( log_dir=args.log_dir, histogram_freq=0, batch_size=args.batch_size, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) callbacks.append(tensorboard_callback) if dataset_type == 'coco': import keras_retinanet.callbacks.coco # use prediction model for evaluation evaluation = keras_retinanet.callbacks.coco.CocoEval( validation_generator) evaluation = RedirectModel(evaluation, prediction_model) callbacks.append(evaluation) lr_scheduler = keras.callbacks.ReduceLROnPlateau( monitor=args.lr_monitor, factor=args.lr_factor, patience=args.lr_patience, verbose=args.lr_verbose, mode=args.lr_mode, min_delta=args.lr_min_delta, cooldown=args.lr_cooldown, min_lr=args.lr_min_lr) callbacks.append(lr_scheduler) return callbacks
def create_callbacks(model, batch_size, weight_file=None, tensorboard_dir=None, snapshots_path=None, backbone=None, dataset_type=None): callbacks = [] if tensorboard_dir: tensorboard_callback = TensorBoard( log_dir=tensorboard_dir, histogram_freq=0, batch_size=batch_size, write_graph=False, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None ) callbacks.append(tensorboard_callback) # save the model if snapshots_path: # ensure directory created first; otherwise h5py will error after epoch. checkpoint = ModelCheckpoint( os.path.join( snapshots_path, '{backbone}_{dataset_type}_{{epoch:02d}}.h5'.format(backbone=backbone, dataset_type=dataset_type) ), verbose=1, # save_best_only=True, # monitor="mAP", # mode='max' ) checkpoint = RedirectModel(checkpoint, model) else: if not weight_file: weight_file = 'retinanet_unnamed.h5' checkpoint = ModelCheckpoint( weight_file, monitor='val_acc', verbose=1, save_best_only=True, save_weights_only=True, mode='auto' ) callbacks.append(checkpoint) callbacks.append(ReduceLROnPlateau( monitor='loss', factor=0.1, patience=2, verbose=1, mode='auto', min_delta=0.0001, cooldown=0, min_lr=0 )) return callbacks
def create_callbacks(model, training_model, prediction_model, validation_generator, dataset_type, snapshot_path): callbacks = [] # save the prediction model checkpoint = keras.callbacks.ModelCheckpoint(os.path.join( snapshot_path, 'circuit_resnet50_{dataset_type}_{{epoch:02d}}.h5'.format( dataset_type=dataset_type)), verbose=1) checkpoint = RedirectModel(checkpoint, prediction_model) callbacks.append(checkpoint) if dataset_type == 'coco': import keras_retinanet.callbacks.coco # use prediction model for evaluation evaluation = keras_retinanet.callbacks.coco.CocoEval( validation_generator) evaluation = RedirectModel(evaluation, prediction_model) callbacks.append(evaluation) lr_scheduler = keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.1, patience=2, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0) callbacks.append(lr_scheduler) tensorboard_callback = keras.callbacks.TensorBoard( log_dir='./logs', histogram_freq=0, batch_size=32, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) callbacks.append(tensorboard_callback) return callbacks
def main(args=None): # parse arguments if args is None: args = sys.argv[1:] args = parse_args(args) # optionally choose specific GPU # if args.gpu: # os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu with tf.device('/device:GPU:3'): config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False, device_count={'CPU': 1, 'GPU': 1}) config.gpu_options.allow_growth = True session = tf.Session(config=config) keras.backend.set_session(session) # create the generators val_generator = create_val_generator(args) weights = args.weights # default to imagenet if nothing else is specified backbone = models.backbone(args.backbone) print('Creating model, this may take a second...') # model = model_with_weights(backbone(val_generator.num_classes(), num_anchors=None, modifier=freeze_model), weights=weights, skip_mismatch=True) model, training_model, prediction_model = create_models( backbone_retinanet=backbone.retinanet, num_classes=val_generator.num_classes(), weights=weights, multi_gpu=args.multi_gpu, freeze_backbone=args.freeze_backbone, lr=args.lr, config=args.config ) # print model summary print(model.summary()) evaluation = Evaluate(val_generator, weighted_average=args.weighted_average, eval_batch_size=args.val_batch_size) evaluation = RedirectModel(evaluation, prediction_model) generator = val_generator iou_threshold = 0.5 score_threshold = 0.05 max_detections = 100 save_path = None weighted_average = False verbose = True eval_batch_size = 16 from keras_retinanet.utils.sail_eval_data_generator import SAIL_EvalDataGenerator eval_batch_datagen = SAIL_EvalDataGenerator(generator, batch_size=eval_batch_size) boxes, scores, labels = model.predict_generator(eval_batch_datagen, steps=len(eval_batch_datagen), verbose=True, workers=0, use_multiprocessing=False)[:3]
def create_callbacks(model, training_model, prediction_model): callbacks = [] # save the prediction model checkpoint = keras.callbacks.ModelCheckpoint(os.path.join( boring_snapshots_dir, 'boring_resnet50_{epoch:02d}.h5'), verbose=1) checkpoint = RedirectModel(checkpoint, prediction_model) callbacks.append(checkpoint) lr_scheduler = keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.1, patience=2, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0) callbacks.append(lr_scheduler) return callbacks
def __create_callbacks(self, model, training_model, prediction_model, tensorboard_dir, snapshot_dir, snapshot_tag, snapshot_freq): callbacks = [] if tensorboard_dir: tensorboard_callback = keras.callbacks.TensorBoard( log_dir=tensorboard_dir, histogram_freq=0, batch_size=self._batch_size, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) callbacks.append(tensorboard_callback) if snapshot_dir: makedirs(snapshot_dir) s_tag = snapshot_tag + "_" if snapshot_tag else "" checkpoint = keras.callbacks.ModelCheckpoint(os.path.join( snapshot_dir, '{tag}{backbone}_{{epoch:02d}}.h5'.format( tag=s_tag, backbone=self._backbone_name)), verbose=1, period=snapshot_freq) checkpoint = RedirectModel(checkpoint, model) callbacks.append(checkpoint) callbacks.append( keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.1, patience=2, verbose=1, mode='auto', min_delta=0.0001, cooldown=0, min_lr=0)) return callbacks
def create_callbacks(model, training_model, prediction_model, validation_generator, args): """ Creates the callbacks to use during training. Args model: The base model. training_model: The model that is used for training. prediction_model: The model that should be used for validation. validation_generator: The generator for creating validation data. args: parseargs args object. Returns: A list of callbacks used for training. """ callbacks = [] tensorboard_callback = None if args.tensorboard_dir: tensorboard_callback = keras.callbacks.TensorBoard( log_dir=args.tensorboard_dir, histogram_freq=0, batch_size=args.batch_size, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) callbacks.append(tensorboard_callback) if args.evaluation and validation_generator: evaluation = Evaluate(validation_generator, score_threshold=args.score_threshold, iou_threshold=args.iou_threshold, tensorboard=tensorboard_callback) evaluation = RedirectModel(evaluation, prediction_model) callbacks.append(evaluation) # save the model if args.snapshots: # ensure directory created first; otherwise h5py will error after epoch. makedirs(args.snapshot_path) checkpoint = keras.callbacks.ModelCheckpoint( os.path.join( args.snapshot_path, '{backbone}_fold_{fold}_last.h5'.format(backbone=args.backbone, fold=args.fold)), verbose=1, ) checkpoint = RedirectModel(checkpoint, model) callbacks.append(checkpoint) checkpoint = keras.callbacks.ModelCheckpoint(os.path.join( args.snapshot_path, '{backbone}_fold_{fold}_{{mAP:.4f}}_{{epoch:02d}}.h5'.format( backbone=args.backbone, fold=args.fold)), verbose=1, save_best_only=False, monitor="mAP", mode='max') checkpoint = RedirectModel(checkpoint, model) callbacks.append(checkpoint) callbacks.append( keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.9, patience=2, verbose=1, mode='auto', min_delta=0.0000001, cooldown=0, min_lr=0)) callbacks.append( keras.callbacks.CSVLogger( args.snapshot_path + '{backbone}_fold_{fold}.csv'.format( backbone=os.path.basename(args.backbone), fold=args.fold), append=True, )) return callbacks
def create_callbacks(model, training_model, prediction_model, validation_generator, args, experiment, DeepForest_config): """ Creates the callbacks to use during training. Args model: The base model. training_model: The model that is used for training. prediction_model: The model that should be used for validation. validation_generator: The generator for creating validation data. args: parseargs args object. Returns: A list of callbacks used for training. """ callbacks = [] tensorboard_callback = None if args.tensorboard_dir: tensorboard_callback = keras.callbacks.TensorBoard( log_dir=args.tensorboard_dir, histogram_freq=0, batch_size=args.batch_size, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) callbacks.append(tensorboard_callback) if args.evaluation and validation_generator: evaluation = Evaluate(validation_generator, tensorboard=tensorboard_callback, experiment=experiment, save_path=args.save_path, score_threshold=args.score_threshold, DeepForest_config=DeepForest_config) evaluation = RedirectModel(evaluation, prediction_model) callbacks.append(evaluation) # save the model if args.snapshots: # ensure directory created first; otherwise h5py will error after epoch. makedirs(args.snapshot_path) checkpoint = keras.callbacks.ModelCheckpoint(os.path.join( args.snapshot_path, '{backbone}_{{epoch:02d}}.h5'.format(backbone=args.backbone)), verbose=1, save_best_only=True, monitor="mAP", mode='max') checkpoint = RedirectModel(checkpoint, model) callbacks.append(checkpoint) callbacks.append( keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.1, patience=2, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0)) #Neon Callbacks site = DeepForest_config["evaluation_site"] #if site=="OSBS": #jaccard=jaccardCallback(validation_generator,DeepForest_config=DeepForest_config,save_path=args.save_path,experiment=experiment) #jaccard = RedirectModel(jaccard, prediction_model) #callbacks.append(jaccard) recall = recallCallback(site=site, generator=validation_generator, save_path=args.save_path, DeepForest_config=DeepForest_config, score_threshold=args.score_threshold, experiment=experiment) recall = RedirectModel(recall, prediction_model) callbacks.append(recall) #Neon mean IoU precision #create the NEON mAP generator NEON_generator = create_NEON_generator(args, site, DeepForest_config) neon_evaluation = NEONmAP(NEON_generator, experiment=experiment, save_path=args.save_path, score_threshold=args.score_threshold, DeepForest_config=DeepForest_config) neon_evaluation = RedirectModel(neon_evaluation, prediction_model) callbacks.append(neon_evaluation) return callbacks
def create_callbacks(model, training_model, prediction_model, validation_generator, args): """ Creates the callbacks to use during training. Args model: The base model. training_model: The model that is used for training. prediction_model: The model that should be used for validation. validation_generator: The generator for creating validation data. args: parseargs args object. Returns: A list of callbacks used for training. """ callbacks = [] tensorboard_callback = None if args.tensorboard_dir: tensorboard_callback = keras.callbacks.TensorBoard( log_dir = args.tensorboard_dir, histogram_freq = 0, batch_size = args.batch_size, write_graph = True, write_grads = False, write_images = False, embeddings_freq = 0, embeddings_layer_names = None, embeddings_metadata = None ) callbacks.append(tensorboard_callback) if args.evaluation and validation_generator: if args.dataset_type == 'coco': from ..callbacks.coco import CocoEval # use prediction model for evaluation evaluation = CocoEval(validation_generator, tensorboard=tensorboard_callback) else: evaluation = Evaluate(validation_generator, tensorboard=tensorboard_callback, weighted_average=args.weighted_average) evaluation = RedirectModel(evaluation, prediction_model) callbacks.append(evaluation) # save the model if args.snapshots: # ensure directory created first; otherwise h5py will error after epoch. makedirs(args.snapshot_path) checkpoint = keras.callbacks.ModelCheckpoint( os.path.join( args.snapshot_path, '{backbone}_{dataset_type}_{{epoch:02d}}.h5'.format(backbone=args.backbone, dataset_type=args.dataset_type) ), verbose=1, save_best_only=False, monitor="mAP", mode='max' ) checkpoint = RedirectModel(checkpoint, model) callbacks.append(checkpoint) callbacks.append(keras.callbacks.ReduceLROnPlateau( monitor = 'loss', factor = 0.9, patience = 4, verbose = 1, mode = 'auto', min_delta = 0.0001, cooldown = 0, min_lr = 0 )) return callbacks
def train_model(concepts, model_name, collectionIds, min_examples, epochs, download_data=True): classmap = get_classmap(concepts) # Downloads the annotation data and saves it into training and validation csv's. # Also downloads corresponding images. if download_data: folders = ["weights"] for dir in folders: if os.path.exists(dir): shutil.rmtree(dir) os.makedirs(dir) start = time.time() print("Starting Download.") download_annotations(min_examples, collectionIds, concepts, classmap, img_folder, train_annot_file, valid_annot_file) end = time.time() print("Done Downloading Annotations: " + str((end - start) / 60) + " minutes") # Trains the model!!!!! WOOOT WOOOT! start = time.time() print("Starting Training.") # Suggested to initialize model on cpu before turning into a # multi_gpu model to save gpu memory with tf.device('/cpu:0'): model = models.backbone('resnet50').retinanet( num_classes=len(concepts)) #modifier=freeze_model) model.load_weights(weights_path, by_name=True, skip_mismatch=True) gpus = len( [i for i in device_lib.list_local_devices() if i.device_type == 'GPU']) if gpus > 1: training_model = multi_gpu_model(model, gpus=gpus) else: training_model = model training_model.compile(loss={ 'regression': losses.smooth_l1(), 'classification': losses.focal() }, optimizer=keras.optimizers.adam(lr=1e-5, clipnorm=0.001)) # transform_generator = random_transform_generator( # min_rotation=-0.1, # max_rotation=0.1, # min_translation=(-0.1, -0.1), # max_translation=(0.1, 0.1), # min_shear=-0.1, # max_shear=0.1, # min_scaling=(0.9, 0.9), # max_scaling=(1.1, 1.1), # flip_x_chance=0.5, # flip_y_chance=0.5, # ) temp = pd.DataFrame(list(zip(classmap.values(), classmap.keys()))) temp.to_csv('classmap.csv', index=False, header=False) train_generator = custom( train_annot_file, 'classmap.csv', # transform_generator=transform_generator, batch_size=batch_size) test_generator = CSVGenerator(valid_annot_file, 'classmap.csv', batch_size=batch_size, shuffle_groups=False) # Checkpoint: save models that are improvements checkpoint = ModelCheckpoint(weights_path, monitor='val_loss', save_best_only=True) checkpoint = RedirectModel(checkpoint, model) #stopping: stops training if val_loss stops improving stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=10) log_table_name = 'previous_runs' # Initialize a log entry in the previous_runs table # TODO: put this in progress_callbacks.py, it makes more sense there # tb_log_id = create_log_entry( # table_name=log_table_name, # model_name=model_name, # users=users, # videos=videos, # min_examples=min_examples, # concepts=selected_concepts, # epochs=epochs # ) # Every epoch upload tensorboard logs to the S3 bucket # log_callback = TensorBoardLog(id_=tb_log_id, table_name=log_table_name) # tensorboard_callback = keras.callbacks.TensorBoard( # log_dir=f'./logs/{tb_log_id}', histogram_freq=0, batch_size=batch_size, # write_graph=True, write_grads=False, write_images=False, # embeddings_freq=0, embeddings_layer_names=None, # embeddings_metadata=None, embeddings_data=None, update_freq='epoch') # Every batch and epoch update a database table with the current progress progress_callback = Progress(steps_per_epoch=len(train_generator), num_epochs=epochs) history = training_model.fit_generator( train_generator, epochs=epochs, callbacks=[checkpoint, stopping, progress_callback], #, log_callback], validation_data=test_generator, verbose=2).history s3.upload_file(weights_path, S3_BUCKET, S3_BUCKET_WEIGHTS_FOLDER + model_name + ".h5") end = time.time() print("Done Training Model: " + str((end - start) / 60) + " minutes")
del X del y if __name__ == '__main__': # model, training_model = get_unet_model() # model, training_model = get_model() model = Resnet3DBuilder.build_resnet_50((64, 64, 16, 1), 2) training_model = keras.utils.multi_gpu_model(model) training_model.compile(optimizer=Adam(amsgrad=True), loss='binary_crossentropy', metrics=['accuracy']) model.summary() callbacks=[ RedirectModel(keras.callbacks.ModelCheckpoint( os.path.join( './model_checkpoints', '{epoch:02d}.h5' ), verbose=1, ), model), keras.callbacks.TensorBoard( log_dir='./logs/' + datetime.datetime.now().strftime('%Y%m%d%H%M') ), # keras.callbacks.ReduceLROnPlateau( # monitor='val_loss', # factor=0.1, # patience=3 # ) ] train_generator = batch_generatorV2('train', 128, n_batch_per_scan=20, negative_ratio=0.8) valid_generator = batch_generatorV2('valid', 128, n_batch_per_scan=20, negative_ratio=0.8)
def create_callbacks(model, training_model, prediction_model, train_generator, validation_generator, args, experiment, DeepForest_config): """ Creates the callbacks to use during training. Args model: The base model. training_model: The model that is used for training. prediction_model: The model that should be used for validation. validation_generator: The generator for creating validation data. args: parseargs args object. Returns: A list of callbacks used for training. """ callbacks = [] if args.evaluation and validation_generator: evaluation = Evaluate(validation_generator, experiment=experiment, save_path=args.save_path, score_threshold=args.score_threshold, DeepForest_config=DeepForest_config, ) evaluation = RedirectModel(evaluation, prediction_model) callbacks.append(evaluation) # save the model if args.snapshots: # ensure directory created first; otherwise h5py will error after epoch. makedirs(args.snapshot_path) checkpoint = keras.callbacks.ModelCheckpoint( os.path.join( args.snapshot_path, '{backbone}_{{epoch:02d}}.h5'.format(backbone=args.backbone) ), verbose=1 #, #save_best_only=True, #monitor="mAP", #mode='max' ) checkpoint = RedirectModel(checkpoint, model) callbacks.append(checkpoint) callbacks.append(keras.callbacks.ReduceLROnPlateau( monitor = 'loss', factor = 0.1, patience = 2, verbose = 1, mode = 'auto', epsilon = 0.0001, cooldown = 0, min_lr = 0 )) #Neon Callbacks NEON_recall_generator = create_NEON_generator(args.batch_size, DeepForest_config) recall = recallCallback( generator=NEON_recall_generator, save_path=args.save_path, score_threshold=args.score_threshold, experiment=experiment, sites=DeepForest_config["evaluation_site"] ) recall = RedirectModel(recall, prediction_model) callbacks.append(recall) #create the NEON generator NEON_generator = create_NEON_generator(args.batch_size, DeepForest_config) neon_evaluation = NEONmAP(NEON_generator, experiment=experiment, save_path=args.save_path, score_threshold=args.score_threshold, DeepForest_config=DeepForest_config) neon_evaluation = RedirectModel(neon_evaluation, prediction_model) callbacks.append(neon_evaluation) return callbacks
tensorboard_callback = keras.callbacks.TensorBoard( log_dir=tensorboard_dir, histogram_freq=0, batch_size=batch_size, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) callbacks.append(tensorboard_callback) if evaluation and validation_generator: evaluation = Evaluate(validation_generator, tensorboard=tensorboard_callback) evaluation = RedirectModel(evaluation, prediction_model) callbacks.append(evaluation) # save the model if snapshots: # ensure directory created first; otherwise h5py will error after epoch. try: os.makedirs(snapshot_path) except OSError: if not os.path.isdir(snapshot_path): raise checkpoint = keras.callbacks.ModelCheckpoint( os.path.join( snapshot_path, '{backbone}_{dataset_type}_{{epoch:02d}}.h5'.format( backbone=backbone, dataset_type='csv')), verbose=1,
def create_callbacks(model, training_model, prediction_model, validation_generator, args): callbacks = [] # save the last prediction model if args.snapshots: # ensure directory created first; otherwise h5py will error after epoch. os.makedirs(args.snapshot_path, exist_ok=True) checkpoint = keras.callbacks.ModelCheckpoint( os.path.join( args.snapshot_path, '{backbone}_fold_{fold}_last.h5'.format(backbone=args.backbone, fold=args.fold)), verbose=1, ) checkpoint = RedirectModel(checkpoint, model) callbacks.append(checkpoint) tensorboard_callback = None if args.tensorboard_dir: tensorboard_callback = keras.callbacks.TensorBoard( log_dir=args.tensorboard_dir, histogram_freq=0, batch_size=args.batch_size, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) callbacks.append(tensorboard_callback) # Calculate mAP if args.evaluation and validation_generator: evaluation = Evaluate(validation_generator, tensorboard=tensorboard_callback, weighted_average=args.weighted_average, save_map_path=args.snapshot_path + '/mask_rcnn_fold_{}.txt'.format(args.fold)) evaluation = RedirectModel(evaluation, prediction_model) callbacks.append(evaluation) # save prediction model with mAP if args.snapshots: checkpoint = keras.callbacks.ModelCheckpoint(os.path.join( args.snapshot_path, '{backbone}_fold_{fold}_{{mAP:.4f}}_ep_{{epoch:02d}}.h5'.format( backbone=args.backbone, fold=args.fold)), verbose=1, save_best_only=False, monitor="mAP", mode='max') checkpoint = RedirectModel(checkpoint, prediction_model) callbacks.append(checkpoint) callbacks.append( keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.9, patience=3, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0)) return callbacks