def create_callbacks(model, training_model, prediction_model,
validation_generator, dataset_type):
callbacks = []
# save the prediction model
checkpoint = keras.callbacks.ModelCheckpoint(os.path.join(
'snapshots', 'resnet50_{dataset_type}_{{epoch:02d}}.h5'.format(
dataset_type=dataset_type)),
verbose=1)
checkpoint = RedirectModel(checkpoint, prediction_model)
callbacks.append(checkpoint)
if dataset_type == 'coco':
import keras_retinanet.callbacks.coco
# use prediction model for evaluation
evaluation = keras_retinanet.callbacks.coco.CocoEval(
validation_generator)
evaluation = RedirectModel(evaluation, prediction_model)
callbacks.append(evaluation)
lr_scheduler = keras.callbacks.ReduceLROnPlateau(monitor='loss',
factor=0.1,
patience=2,
verbose=1,
mode='auto',
epsilon=0.0001,
cooldown=0,
min_lr=0)
callbacks.append(lr_scheduler)
return callbacks
def create_callbacks(model, training_model, prediction_model, train_generator, validation_generator, p):
callbacks = []
# save the prediction model
if p['snapshots']:
checkpoint = keras.callbacks.ModelCheckpoint(
os.path.join(
p['snapshot-path'],
'resnet'+str(p['resnet'])+'_{epoch:02d}.h5'
),
verbose=1
)
checkpoint = RedirectModel(checkpoint, prediction_model)
callbacks.append(checkpoint)
# from ..callbacks.coco import CocoEval
# evaluation = CocoEval(validation_generator)
# callbacks.append(evaluation)
if p['lr-plateau'] == True:
lr_scheduler = keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.1, patience=2, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0)
callbacks.append(lr_scheduler)
if p['batch-log'] == True:
global batch_log_file
batch_log_file = open(p['batch-log-name'],'a')
logging_callback = keras.callbacks.LambdaCallback(
on_batch_end = lambda batch, logs: batch_log_file.write('{},{}\n'.format(batch,logs['loss']))
)
callbacks.append(logging_callback)
if p['loss'] == True:
global valid_loss_file
global train_loss_file
valid_loss_file = open(p['loss-folder-path']+'valid-loss.csv','a')
train_loss_file = open(p['loss-folder-path']+'train-loss.csv','a')
valid_loss_callback = keras.callbacks.LambdaCallback(
on_epoch_end = lambda epoch, logs: valid_loss_file.write('{},{}\n'.format(epoch,logs['val_loss']))
)
train_loss_callback = keras.callbacks.LambdaCallback(
on_epoch_end = lambda epoch, logs: train_loss_file.write('{},{}\n'.format(epoch,logs['loss']))
)
callbacks.append(valid_loss_callback)
callbacks.append(train_loss_callback)
if p['valid-acc'] == True:
evaluation = LogoEval(validation_generator,filepath=p['valid-acc-path'],threshold=p['threshold'],iou_threshold=p['iou-threshold'])
evaluation = RedirectModel(evaluation, prediction_model)
callbacks.append(evaluation)
if p['train-acc'] == True:
evaluation = LogoEval(train_generator,filepath=p['train-acc-path'],threshold=p['threshold'],iou_threshold=p['iou-threshold'])
evaluation = RedirectModel(evaluation, prediction_model)
callbacks.append(evaluation)
return callbacks
def create_callbacks(model, training_model, prediction_model, validation_generator, args, create_evaluation=Evaluate):
callbacks = []
# save the prediction model
if args.snapshots:
# ensure directory created first; otherwise h5py will error after epoch.
os.makedirs(args.snapshot_path, exist_ok=True)
checkpoint = keras.callbacks.ModelCheckpoint(
os.path.join(
args.snapshot_path,
'{backbone}_{dataset_type}_{{epoch:02d}}.h5'.format(backbone=args.backbone, dataset_type=args.dataset_type)
),
verbose=1
)
checkpoint = RedirectModel(checkpoint, model)
callbacks.append(checkpoint)
tensorboard_callback = None
if args.tensorboard_dir:
tensorboard_callback = keras.callbacks.TensorBoard(
log_dir = args.tensorboard_dir,
histogram_freq = 0,
batch_size = args.batch_size,
write_graph = True,
write_grads = False,
write_images = False,
embeddings_freq = 0,
embeddings_layer_names = None,
embeddings_metadata = None
)
callbacks.append(tensorboard_callback)
if args.evaluation and validation_generator:
if args.dataset_type == 'coco':
from ..callbacks.coco import CocoEval
# use prediction model for evaluation
evaluation = CocoEval(validation_generator)
elif create_evaluation:
evaluation = create_evaluation(validation_generator, tensorboard=tensorboard_callback, weighted_average=args.weighted_average)
else:
evaluation = Evaluate(validation_generator, tensorboard=tensorboard_callback, weighted_average=args.weighted_average)
evaluation = RedirectModel(evaluation, prediction_model)
callbacks.append(evaluation)
callbacks.append(keras.callbacks.ReduceLROnPlateau(
monitor = 'loss',
factor = 0.1,
patience = 2,
verbose = 1,
mode = 'auto',
epsilon = 0.0001,
cooldown = 0,
min_lr = 0
))
return callbacks
def create_callbacks(model, training_model, prediction_model,
validation_generator, args, comet_experiment):
"""Creates the callbacks to use during training.
Args
model: The base model.
training_model: The model that is used for training.
prediction_model: The model that should be used for validation.
validation_generator: The generator for creating validation data.
args: parseargs args object.
comet_experiment: cometml object to log images
Returns:
A list of callbacks used for training.
"""
callbacks = []
if args.evaluation and validation_generator:
evaluation = Evaluate(validation_generator,
weighted_average=args.weighted_average,
comet_experiment=comet_experiment,
save_path=args.save_path,
score_threshold=args.score_threshold)
evaluation = RedirectModel(evaluation, prediction_model)
callbacks.append(evaluation)
# save the model
if args.snapshots:
# ensure directory created first; otherwise h5py will error after epoch.
makedirs(args.snapshot_path)
checkpoint = keras.callbacks.ModelCheckpoint(
os.path.join(
args.snapshot_path,
'{backbone}_{dataset_type}_{{epoch:02d}}.h5'.format(
backbone=args.backbone, dataset_type=args.dataset_type)),
verbose=1,
# save_best_only=True,
# monitor="mAP",
# mode='max'
)
checkpoint = RedirectModel(checkpoint, model)
callbacks.append(checkpoint)
callbacks.append(
keras.callbacks.ReduceLROnPlateau(monitor='loss',
factor=0.1,
patience=2,
verbose=1,
mode='auto',
min_delta=0.0001,
cooldown=0,
min_lr=0))
return callbacks
def create_callbacks(model, training_model, prediction_model,
validation_generator, args):
callbacks = []
# save the prediction model
if args.snapshots:
# ensure directory created first; otherwise h5py will error after epoch.
io_utils.create_folder(args.snapshot_path)
checkpoint = keras.callbacks.ModelCheckpoint(os.path.join(
args.snapshot_path,
'{backbone}_{dataset_type}_{{epoch:02d}}.h5'.format(
backbone=args.backbone, dataset_type=args.dataset_type)),
verbose=1)
checkpoint = RedirectModel(checkpoint, prediction_model)
callbacks.append(checkpoint)
tensorboard_callback = None
if args.tensorboard_dir:
tensorboard_callback = keras.callbacks.TensorBoard(
log_dir=args.tensorboard_dir,
histogram_freq=0,
batch_size=args.batch_size,
write_graph=True,
write_grads=False,
write_images=False,
embeddings_freq=0,
embeddings_layer_names=None,
embeddings_metadata=None)
callbacks.append(tensorboard_callback)
if args.evaluation and validation_generator:
if args.dataset_type == 'traffic_signs':
# use prediction model for evaluation
ground_truth_proto_file = os.path.join(args.val_path, 'rois.bin')
evaluation = TrafficSignsEval(
validation_generator, ground_truth_proto_file,
os.path.join(args.train_path, 'rois.bin'))
evaluation = RedirectModel(evaluation, prediction_model)
callbacks.append(evaluation)
callbacks.append(
keras.callbacks.ReduceLROnPlateau(monitor='loss',
factor=0.1,
patience=2,
verbose=1,
mode='auto',
epsilon=0.0001,
cooldown=0,
min_lr=0))
return callbacks
def create_callbacks(model, args):
""" Creates the callbacks to use during training.
Args
model: The base model.
training_model: The model that is used for training.
prediction_model: The model that should be used for validation.
validation_generator: The generator for creating validation data.
args: parseargs args object.
Returns:
A list of callbacks used for training.
"""
callbacks = []
tensorboard_callback = None
if args.tensorboard_dir:
tensorboard_callback = keras.callbacks.TensorBoard(
log_dir=args.tensorboard_dir,
histogram_freq=0,
batch_size=args.batch_size,
write_graph=True,
write_grads=False,
write_images=False,
embeddings_freq=0,
embeddings_layer_names=None,
embeddings_metadata=None)
callbacks.append(tensorboard_callback)
# save the model
if args.snapshots:
# ensure directory created first; otherwise h5py will error after epoch.
makedirs(args.snapshot_path)
checkpoint = keras.callbacks.ModelCheckpoint(
os.path.join(
args.snapshot_path,
'{backbone}_{dataset_type}_level_{level}_{{epoch:02d}}.h5'.
format(backbone=args.backbone,
level=args.label_level,
dataset_type=args.dataset_type)),
verbose=1,
# save_best_only=True,
# monitor="mAP",
# mode='max'
)
checkpoint = RedirectModel(checkpoint, model)
callbacks.append(checkpoint)
callbacks.append(
keras.callbacks.ReduceLROnPlateau(
monitor='loss',
factor=0.8,
patience=2,
verbose=1,
mode='auto',
cooldown=0,
min_lr=5e-6,
))
return callbacks
def create_callbacks(model, training_model, prediction_model,
validation_generator, dataset_type, snapshot_path, args):
callbacks = []
# save the prediction model
checkpoint = keras.callbacks.ModelCheckpoint(os.path.join(
snapshot_path, '{backbone}_{dataset_type}_{{epoch:02d}}.h5'.format(
backbone=args.backbone, dataset_type=dataset_type)),
verbose=1)
checkpoint = RedirectModel(checkpoint, prediction_model)
callbacks.append(checkpoint)
if args.log_dir:
tensorboard_callback = keras.callbacks.TensorBoard(
log_dir=args.log_dir,
histogram_freq=0,
batch_size=args.batch_size,
write_graph=True,
write_grads=False,
write_images=False,
embeddings_freq=0,
embeddings_layer_names=None,
embeddings_metadata=None)
callbacks.append(tensorboard_callback)
if dataset_type == 'coco':
import keras_retinanet.callbacks.coco
# use prediction model for evaluation
evaluation = keras_retinanet.callbacks.coco.CocoEval(
validation_generator)
evaluation = RedirectModel(evaluation, prediction_model)
callbacks.append(evaluation)
lr_scheduler = keras.callbacks.ReduceLROnPlateau(
monitor=args.lr_monitor,
factor=args.lr_factor,
patience=args.lr_patience,
verbose=args.lr_verbose,
mode=args.lr_mode,
min_delta=args.lr_min_delta,
cooldown=args.lr_cooldown,
min_lr=args.lr_min_lr)
callbacks.append(lr_scheduler)
return callbacks
def create_callbacks(model, batch_size, weight_file=None, tensorboard_dir=None, snapshots_path=None,
backbone=None, dataset_type=None):
callbacks = []
if tensorboard_dir:
tensorboard_callback = TensorBoard(
log_dir=tensorboard_dir,
histogram_freq=0,
batch_size=batch_size,
write_graph=False,
write_grads=False,
write_images=False,
embeddings_freq=0,
embeddings_layer_names=None,
embeddings_metadata=None
)
callbacks.append(tensorboard_callback)
# save the model
if snapshots_path:
# ensure directory created first; otherwise h5py will error after epoch.
checkpoint = ModelCheckpoint(
os.path.join(
snapshots_path,
'{backbone}_{dataset_type}_{{epoch:02d}}.h5'.format(backbone=backbone,
dataset_type=dataset_type)
),
verbose=1,
# save_best_only=True,
# monitor="mAP",
# mode='max'
)
checkpoint = RedirectModel(checkpoint, model)
else:
if not weight_file:
weight_file = 'retinanet_unnamed.h5'
checkpoint = ModelCheckpoint(
weight_file,
monitor='val_acc',
verbose=1,
save_best_only=True,
save_weights_only=True,
mode='auto'
)
callbacks.append(checkpoint)
callbacks.append(ReduceLROnPlateau(
monitor='loss',
factor=0.1,
patience=2,
verbose=1,
mode='auto',
min_delta=0.0001,
cooldown=0,
min_lr=0
))
return callbacks
def create_callbacks(model, training_model, prediction_model,
validation_generator, dataset_type, snapshot_path):
callbacks = []
# save the prediction model
checkpoint = keras.callbacks.ModelCheckpoint(os.path.join(
snapshot_path,
'circuit_resnet50_{dataset_type}_{{epoch:02d}}.h5'.format(
dataset_type=dataset_type)),
verbose=1)
checkpoint = RedirectModel(checkpoint, prediction_model)
callbacks.append(checkpoint)
if dataset_type == 'coco':
import keras_retinanet.callbacks.coco
# use prediction model for evaluation
evaluation = keras_retinanet.callbacks.coco.CocoEval(
validation_generator)
evaluation = RedirectModel(evaluation, prediction_model)
callbacks.append(evaluation)
lr_scheduler = keras.callbacks.ReduceLROnPlateau(monitor='loss',
factor=0.1,
patience=2,
verbose=1,
mode='auto',
epsilon=0.0001,
cooldown=0,
min_lr=0)
callbacks.append(lr_scheduler)
tensorboard_callback = keras.callbacks.TensorBoard(
log_dir='./logs',
histogram_freq=0,
batch_size=32,
write_graph=True,
write_grads=False,
write_images=False,
embeddings_freq=0,
embeddings_layer_names=None,
embeddings_metadata=None)
callbacks.append(tensorboard_callback)
return callbacks
def main(args=None):
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
# optionally choose specific GPU
# if args.gpu:
# os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
with tf.device('/device:GPU:3'):
config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False, device_count={'CPU': 1, 'GPU': 1})
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
keras.backend.set_session(session)
# create the generators
val_generator = create_val_generator(args)
weights = args.weights
# default to imagenet if nothing else is specified
backbone = models.backbone(args.backbone)
print('Creating model, this may take a second...')
# model = model_with_weights(backbone(val_generator.num_classes(), num_anchors=None, modifier=freeze_model), weights=weights, skip_mismatch=True)
model, training_model, prediction_model = create_models(
backbone_retinanet=backbone.retinanet,
num_classes=val_generator.num_classes(),
weights=weights,
multi_gpu=args.multi_gpu,
freeze_backbone=args.freeze_backbone,
lr=args.lr,
config=args.config
)
# print model summary
print(model.summary())
evaluation = Evaluate(val_generator, weighted_average=args.weighted_average, eval_batch_size=args.val_batch_size)
evaluation = RedirectModel(evaluation, prediction_model)
generator = val_generator
iou_threshold = 0.5
score_threshold = 0.05
max_detections = 100
save_path = None
weighted_average = False
verbose = True
eval_batch_size = 16
from keras_retinanet.utils.sail_eval_data_generator import SAIL_EvalDataGenerator
eval_batch_datagen = SAIL_EvalDataGenerator(generator, batch_size=eval_batch_size)
boxes, scores, labels = model.predict_generator(eval_batch_datagen, steps=len(eval_batch_datagen), verbose=True,
workers=0, use_multiprocessing=False)[:3]
def create_callbacks(model, training_model, prediction_model):
callbacks = []
# save the prediction model
checkpoint = keras.callbacks.ModelCheckpoint(os.path.join(
boring_snapshots_dir, 'boring_resnet50_{epoch:02d}.h5'),
verbose=1)
checkpoint = RedirectModel(checkpoint, prediction_model)
callbacks.append(checkpoint)
lr_scheduler = keras.callbacks.ReduceLROnPlateau(monitor='loss',
factor=0.1,
patience=2,
verbose=1,
mode='auto',
epsilon=0.0001,
cooldown=0,
min_lr=0)
callbacks.append(lr_scheduler)
return callbacks
def __create_callbacks(self, model, training_model, prediction_model,
tensorboard_dir, snapshot_dir, snapshot_tag,
snapshot_freq):
callbacks = []
if tensorboard_dir:
tensorboard_callback = keras.callbacks.TensorBoard(
log_dir=tensorboard_dir,
histogram_freq=0,
batch_size=self._batch_size,
write_graph=True,
write_grads=False,
write_images=False,
embeddings_freq=0,
embeddings_layer_names=None,
embeddings_metadata=None)
callbacks.append(tensorboard_callback)
if snapshot_dir:
makedirs(snapshot_dir)
s_tag = snapshot_tag + "_" if snapshot_tag else ""
checkpoint = keras.callbacks.ModelCheckpoint(os.path.join(
snapshot_dir, '{tag}{backbone}_{{epoch:02d}}.h5'.format(
tag=s_tag, backbone=self._backbone_name)),
verbose=1,
period=snapshot_freq)
checkpoint = RedirectModel(checkpoint, model)
callbacks.append(checkpoint)
callbacks.append(
keras.callbacks.ReduceLROnPlateau(monitor='loss',
factor=0.1,
patience=2,
verbose=1,
mode='auto',
min_delta=0.0001,
cooldown=0,
min_lr=0))
return callbacks
def create_callbacks(model, training_model, prediction_model,
validation_generator, args):
""" Creates the callbacks to use during training.
Args
model: The base model.
training_model: The model that is used for training.
prediction_model: The model that should be used for validation.
validation_generator: The generator for creating validation data.
args: parseargs args object.
Returns:
A list of callbacks used for training.
"""
callbacks = []
tensorboard_callback = None
if args.tensorboard_dir:
tensorboard_callback = keras.callbacks.TensorBoard(
log_dir=args.tensorboard_dir,
histogram_freq=0,
batch_size=args.batch_size,
write_graph=True,
write_grads=False,
write_images=False,
embeddings_freq=0,
embeddings_layer_names=None,
embeddings_metadata=None)
callbacks.append(tensorboard_callback)
if args.evaluation and validation_generator:
evaluation = Evaluate(validation_generator,
score_threshold=args.score_threshold,
iou_threshold=args.iou_threshold,
tensorboard=tensorboard_callback)
evaluation = RedirectModel(evaluation, prediction_model)
callbacks.append(evaluation)
# save the model
if args.snapshots:
# ensure directory created first; otherwise h5py will error after epoch.
makedirs(args.snapshot_path)
checkpoint = keras.callbacks.ModelCheckpoint(
os.path.join(
args.snapshot_path,
'{backbone}_fold_{fold}_last.h5'.format(backbone=args.backbone,
fold=args.fold)),
verbose=1,
)
checkpoint = RedirectModel(checkpoint, model)
callbacks.append(checkpoint)
checkpoint = keras.callbacks.ModelCheckpoint(os.path.join(
args.snapshot_path,
'{backbone}_fold_{fold}_{{mAP:.4f}}_{{epoch:02d}}.h5'.format(
backbone=args.backbone, fold=args.fold)),
verbose=1,
save_best_only=False,
monitor="mAP",
mode='max')
checkpoint = RedirectModel(checkpoint, model)
callbacks.append(checkpoint)
callbacks.append(
keras.callbacks.ReduceLROnPlateau(monitor='loss',
factor=0.9,
patience=2,
verbose=1,
mode='auto',
min_delta=0.0000001,
cooldown=0,
min_lr=0))
callbacks.append(
keras.callbacks.CSVLogger(
args.snapshot_path + '{backbone}_fold_{fold}.csv'.format(
backbone=os.path.basename(args.backbone), fold=args.fold),
append=True,
))
return callbacks
def create_callbacks(model, training_model, prediction_model,
validation_generator, args, experiment,
DeepForest_config):
""" Creates the callbacks to use during training.
Args
model: The base model.
training_model: The model that is used for training.
prediction_model: The model that should be used for validation.
validation_generator: The generator for creating validation data.
args: parseargs args object.
Returns:
A list of callbacks used for training.
"""
callbacks = []
tensorboard_callback = None
if args.tensorboard_dir:
tensorboard_callback = keras.callbacks.TensorBoard(
log_dir=args.tensorboard_dir,
histogram_freq=0,
batch_size=args.batch_size,
write_graph=True,
write_grads=False,
write_images=False,
embeddings_freq=0,
embeddings_layer_names=None,
embeddings_metadata=None)
callbacks.append(tensorboard_callback)
if args.evaluation and validation_generator:
evaluation = Evaluate(validation_generator,
tensorboard=tensorboard_callback,
experiment=experiment,
save_path=args.save_path,
score_threshold=args.score_threshold,
DeepForest_config=DeepForest_config)
evaluation = RedirectModel(evaluation, prediction_model)
callbacks.append(evaluation)
# save the model
if args.snapshots:
# ensure directory created first; otherwise h5py will error after epoch.
makedirs(args.snapshot_path)
checkpoint = keras.callbacks.ModelCheckpoint(os.path.join(
args.snapshot_path,
'{backbone}_{{epoch:02d}}.h5'.format(backbone=args.backbone)),
verbose=1,
save_best_only=True,
monitor="mAP",
mode='max')
checkpoint = RedirectModel(checkpoint, model)
callbacks.append(checkpoint)
callbacks.append(
keras.callbacks.ReduceLROnPlateau(monitor='loss',
factor=0.1,
patience=2,
verbose=1,
mode='auto',
epsilon=0.0001,
cooldown=0,
min_lr=0))
#Neon Callbacks
site = DeepForest_config["evaluation_site"]
#if site=="OSBS":
#jaccard=jaccardCallback(validation_generator,DeepForest_config=DeepForest_config,save_path=args.save_path,experiment=experiment)
#jaccard = RedirectModel(jaccard, prediction_model)
#callbacks.append(jaccard)
recall = recallCallback(site=site,
generator=validation_generator,
save_path=args.save_path,
DeepForest_config=DeepForest_config,
score_threshold=args.score_threshold,
experiment=experiment)
recall = RedirectModel(recall, prediction_model)
callbacks.append(recall)
#Neon mean IoU precision
#create the NEON mAP generator
NEON_generator = create_NEON_generator(args, site, DeepForest_config)
neon_evaluation = NEONmAP(NEON_generator,
experiment=experiment,
save_path=args.save_path,
score_threshold=args.score_threshold,
DeepForest_config=DeepForest_config)
neon_evaluation = RedirectModel(neon_evaluation, prediction_model)
callbacks.append(neon_evaluation)
return callbacks
def create_callbacks(model, training_model, prediction_model, validation_generator, args):
""" Creates the callbacks to use during training.
Args
model: The base model.
training_model: The model that is used for training.
prediction_model: The model that should be used for validation.
validation_generator: The generator for creating validation data.
args: parseargs args object.
Returns:
A list of callbacks used for training.
"""
callbacks = []
tensorboard_callback = None
if args.tensorboard_dir:
tensorboard_callback = keras.callbacks.TensorBoard(
log_dir = args.tensorboard_dir,
histogram_freq = 0,
batch_size = args.batch_size,
write_graph = True,
write_grads = False,
write_images = False,
embeddings_freq = 0,
embeddings_layer_names = None,
embeddings_metadata = None
)
callbacks.append(tensorboard_callback)
if args.evaluation and validation_generator:
if args.dataset_type == 'coco':
from ..callbacks.coco import CocoEval
# use prediction model for evaluation
evaluation = CocoEval(validation_generator, tensorboard=tensorboard_callback)
else:
evaluation = Evaluate(validation_generator, tensorboard=tensorboard_callback, weighted_average=args.weighted_average)
evaluation = RedirectModel(evaluation, prediction_model)
callbacks.append(evaluation)
# save the model
if args.snapshots:
# ensure directory created first; otherwise h5py will error after epoch.
makedirs(args.snapshot_path)
checkpoint = keras.callbacks.ModelCheckpoint(
os.path.join(
args.snapshot_path,
'{backbone}_{dataset_type}_{{epoch:02d}}.h5'.format(backbone=args.backbone, dataset_type=args.dataset_type)
),
verbose=1,
save_best_only=False,
monitor="mAP",
mode='max'
)
checkpoint = RedirectModel(checkpoint, model)
callbacks.append(checkpoint)
callbacks.append(keras.callbacks.ReduceLROnPlateau(
monitor = 'loss',
factor = 0.9,
patience = 4,
verbose = 1,
mode = 'auto',
min_delta = 0.0001,
cooldown = 0,
min_lr = 0
))
return callbacks
def train_model(concepts,
model_name,
collectionIds,
min_examples,
epochs,
download_data=True):
classmap = get_classmap(concepts)
# Downloads the annotation data and saves it into training and validation csv's.
# Also downloads corresponding images.
if download_data:
folders = ["weights"]
for dir in folders:
if os.path.exists(dir):
shutil.rmtree(dir)
os.makedirs(dir)
start = time.time()
print("Starting Download.")
download_annotations(min_examples, collectionIds, concepts, classmap,
img_folder, train_annot_file, valid_annot_file)
end = time.time()
print("Done Downloading Annotations: " + str((end - start) / 60) +
" minutes")
# Trains the model!!!!! WOOOT WOOOT!
start = time.time()
print("Starting Training.")
# Suggested to initialize model on cpu before turning into a
# multi_gpu model to save gpu memory
with tf.device('/cpu:0'):
model = models.backbone('resnet50').retinanet(
num_classes=len(concepts)) #modifier=freeze_model)
model.load_weights(weights_path, by_name=True, skip_mismatch=True)
gpus = len(
[i for i in device_lib.list_local_devices() if i.device_type == 'GPU'])
if gpus > 1:
training_model = multi_gpu_model(model, gpus=gpus)
else:
training_model = model
training_model.compile(loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=1e-5,
clipnorm=0.001))
# transform_generator = random_transform_generator(
# min_rotation=-0.1,
# max_rotation=0.1,
# min_translation=(-0.1, -0.1),
# max_translation=(0.1, 0.1),
# min_shear=-0.1,
# max_shear=0.1,
# min_scaling=(0.9, 0.9),
# max_scaling=(1.1, 1.1),
# flip_x_chance=0.5,
# flip_y_chance=0.5,
# )
temp = pd.DataFrame(list(zip(classmap.values(), classmap.keys())))
temp.to_csv('classmap.csv', index=False, header=False)
train_generator = custom(
train_annot_file,
'classmap.csv',
# transform_generator=transform_generator,
batch_size=batch_size)
test_generator = CSVGenerator(valid_annot_file,
'classmap.csv',
batch_size=batch_size,
shuffle_groups=False)
# Checkpoint: save models that are improvements
checkpoint = ModelCheckpoint(weights_path,
monitor='val_loss',
save_best_only=True)
checkpoint = RedirectModel(checkpoint, model)
#stopping: stops training if val_loss stops improving
stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=10)
log_table_name = 'previous_runs'
# Initialize a log entry in the previous_runs table
# TODO: put this in progress_callbacks.py, it makes more sense there
# tb_log_id = create_log_entry(
# table_name=log_table_name,
# model_name=model_name,
# users=users,
# videos=videos,
# min_examples=min_examples,
# concepts=selected_concepts,
# epochs=epochs
# )
# Every epoch upload tensorboard logs to the S3 bucket
# log_callback = TensorBoardLog(id_=tb_log_id, table_name=log_table_name)
# tensorboard_callback = keras.callbacks.TensorBoard(
# log_dir=f'./logs/{tb_log_id}', histogram_freq=0, batch_size=batch_size,
# write_graph=True, write_grads=False, write_images=False,
# embeddings_freq=0, embeddings_layer_names=None,
# embeddings_metadata=None, embeddings_data=None, update_freq='epoch')
# Every batch and epoch update a database table with the current progress
progress_callback = Progress(steps_per_epoch=len(train_generator),
num_epochs=epochs)
history = training_model.fit_generator(
train_generator,
epochs=epochs,
callbacks=[checkpoint, stopping, progress_callback], #, log_callback],
validation_data=test_generator,
verbose=2).history
s3.upload_file(weights_path, S3_BUCKET,
S3_BUCKET_WEIGHTS_FOLDER + model_name + ".h5")
end = time.time()
print("Done Training Model: " + str((end - start) / 60) + " minutes")
del X
del y
if __name__ == '__main__':
# model, training_model = get_unet_model()
# model, training_model = get_model()
model = Resnet3DBuilder.build_resnet_50((64, 64, 16, 1), 2)
training_model = keras.utils.multi_gpu_model(model)
training_model.compile(optimizer=Adam(amsgrad=True), loss='binary_crossentropy', metrics=['accuracy'])
model.summary()
callbacks=[
RedirectModel(keras.callbacks.ModelCheckpoint(
os.path.join(
'./model_checkpoints',
'{epoch:02d}.h5'
),
verbose=1,
), model),
keras.callbacks.TensorBoard(
log_dir='./logs/' + datetime.datetime.now().strftime('%Y%m%d%H%M')
),
# keras.callbacks.ReduceLROnPlateau(
# monitor='val_loss',
# factor=0.1,
# patience=3
# )
]
train_generator = batch_generatorV2('train', 128, n_batch_per_scan=20, negative_ratio=0.8)
valid_generator = batch_generatorV2('valid', 128, n_batch_per_scan=20, negative_ratio=0.8)
def create_callbacks(model, training_model, prediction_model, train_generator, validation_generator, args, experiment, DeepForest_config):
""" Creates the callbacks to use during training.
Args
model: The base model.
training_model: The model that is used for training.
prediction_model: The model that should be used for validation.
validation_generator: The generator for creating validation data.
args: parseargs args object.
Returns:
A list of callbacks used for training.
"""
callbacks = []
if args.evaluation and validation_generator:
evaluation = Evaluate(validation_generator,
experiment=experiment,
save_path=args.save_path,
score_threshold=args.score_threshold,
DeepForest_config=DeepForest_config,
)
evaluation = RedirectModel(evaluation, prediction_model)
callbacks.append(evaluation)
# save the model
if args.snapshots:
# ensure directory created first; otherwise h5py will error after epoch.
makedirs(args.snapshot_path)
checkpoint = keras.callbacks.ModelCheckpoint(
os.path.join(
args.snapshot_path,
'{backbone}_{{epoch:02d}}.h5'.format(backbone=args.backbone)
),
verbose=1
#,
#save_best_only=True,
#monitor="mAP",
#mode='max'
)
checkpoint = RedirectModel(checkpoint, model)
callbacks.append(checkpoint)
callbacks.append(keras.callbacks.ReduceLROnPlateau(
monitor = 'loss',
factor = 0.1,
patience = 2,
verbose = 1,
mode = 'auto',
epsilon = 0.0001,
cooldown = 0,
min_lr = 0
))
#Neon Callbacks
NEON_recall_generator = create_NEON_generator(args.batch_size, DeepForest_config)
recall = recallCallback(
generator=NEON_recall_generator,
save_path=args.save_path,
score_threshold=args.score_threshold,
experiment=experiment,
sites=DeepForest_config["evaluation_site"] )
recall = RedirectModel(recall, prediction_model)
callbacks.append(recall)
#create the NEON generator
NEON_generator = create_NEON_generator(args.batch_size, DeepForest_config)
neon_evaluation = NEONmAP(NEON_generator,
experiment=experiment,
save_path=args.save_path,
score_threshold=args.score_threshold,
DeepForest_config=DeepForest_config)
neon_evaluation = RedirectModel(neon_evaluation, prediction_model)
callbacks.append(neon_evaluation)
return callbacks
tensorboard_callback = keras.callbacks.TensorBoard(
log_dir=tensorboard_dir,
histogram_freq=0,
batch_size=batch_size,
write_graph=True,
write_grads=False,
write_images=False,
embeddings_freq=0,
embeddings_layer_names=None,
embeddings_metadata=None)
callbacks.append(tensorboard_callback)
if evaluation and validation_generator:
evaluation = Evaluate(validation_generator,
tensorboard=tensorboard_callback)
evaluation = RedirectModel(evaluation, prediction_model)
callbacks.append(evaluation)
# save the model
if snapshots:
# ensure directory created first; otherwise h5py will error after epoch.
try:
os.makedirs(snapshot_path)
except OSError:
if not os.path.isdir(snapshot_path):
raise
checkpoint = keras.callbacks.ModelCheckpoint(
os.path.join(
snapshot_path, '{backbone}_{dataset_type}_{{epoch:02d}}.h5'.format(
backbone=backbone, dataset_type='csv')),
verbose=1,
def create_callbacks(model, training_model, prediction_model,
validation_generator, args):
callbacks = []
# save the last prediction model
if args.snapshots:
# ensure directory created first; otherwise h5py will error after epoch.
os.makedirs(args.snapshot_path, exist_ok=True)
checkpoint = keras.callbacks.ModelCheckpoint(
os.path.join(
args.snapshot_path,
'{backbone}_fold_{fold}_last.h5'.format(backbone=args.backbone,
fold=args.fold)),
verbose=1,
)
checkpoint = RedirectModel(checkpoint, model)
callbacks.append(checkpoint)
tensorboard_callback = None
if args.tensorboard_dir:
tensorboard_callback = keras.callbacks.TensorBoard(
log_dir=args.tensorboard_dir,
histogram_freq=0,
batch_size=args.batch_size,
write_graph=True,
write_grads=False,
write_images=False,
embeddings_freq=0,
embeddings_layer_names=None,
embeddings_metadata=None)
callbacks.append(tensorboard_callback)
# Calculate mAP
if args.evaluation and validation_generator:
evaluation = Evaluate(validation_generator,
tensorboard=tensorboard_callback,
weighted_average=args.weighted_average,
save_map_path=args.snapshot_path +
'/mask_rcnn_fold_{}.txt'.format(args.fold))
evaluation = RedirectModel(evaluation, prediction_model)
callbacks.append(evaluation)
# save prediction model with mAP
if args.snapshots:
checkpoint = keras.callbacks.ModelCheckpoint(os.path.join(
args.snapshot_path,
'{backbone}_fold_{fold}_{{mAP:.4f}}_ep_{{epoch:02d}}.h5'.format(
backbone=args.backbone, fold=args.fold)),
verbose=1,
save_best_only=False,
monitor="mAP",
mode='max')
checkpoint = RedirectModel(checkpoint, prediction_model)
callbacks.append(checkpoint)
callbacks.append(
keras.callbacks.ReduceLROnPlateau(monitor='loss',
factor=0.9,
patience=3,
verbose=1,
mode='auto',
epsilon=0.0001,
cooldown=0,
min_lr=0))
return callbacks
