def create_models(backbone_retinanet,
num_classes,
weights,
multi_gpu=0,
freeze_backbone=False):
modifier = freeze_model if freeze_backbone else None
# Keras recommends initialising a multi-gpu model on the CPU to ease weight sharing, and to prevent OOM errors.
# optionally wrap in a parallel model
if multi_gpu > 1:
with tf.device('/cpu:0'):
model = model_with_weights(backbone_retinanet(num_classes,
modifier=modifier),
weights=weights,
skip_mismatch=True)
training_model = multi_gpu_model(model, gpus=multi_gpu)
else:
model = model_with_weights(backbone_retinanet(num_classes,
modifier=modifier),
weights=weights,
skip_mismatch=True)
training_model = model
# make prediction model
prediction_model = retinanet_bbox(model=model, class_specific_filter=False)
# compile model
training_model.compile(loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=1e-5,
clipnorm=0.001))
return model, training_model, prediction_model
def load_model(filepath,
backbone_name='resnet50',
convert=False,
nms=True,
class_specific_filter=True):
"""Loads a retinanet model using the correct custom objects.
# Arguments
filepath: one of the following:
- string, path to the saved model, or
- h5py.File object from which to load the model
backbone_name: Backbone with which the model was trained.
convert: Boolean, whether to convert the model to an inference model.
nms: Boolean, whether to add NMS filtering to the converted model.
Only valid if convert=True.
class_specific_filter: Whether to use class specific filtering or filter
for the best scoring class only.
# Returns
A keras.models.Model object.
# Raises
ImportError: if h5py is not available.
ValueError: In case of an invalid savefile.
"""
from keras.models import load_model as keras_load_model
model = keras_load_model(
filepath, custom_objects=get_backbone(backbone_name).custom_objects)
if convert:
model = retinanet_bbox(model=model,
nms=nms,
class_specific_filter=class_specific_filter)
return model
def get_RetinaNet_model():
from keras.utils import custom_object_scope
from keras_resnet.layers import BatchNormalization
from keras_retinanet.layers import UpsampleLike, Anchors, RegressBoxes, ClipBoxes, FilterDetections
from keras_retinanet.initializers import PriorProbability
from keras_retinanet import models
from keras_retinanet.models.retinanet import retinanet_bbox
custom_objects = {
'BatchNormalization': BatchNormalization,
'UpsampleLike': UpsampleLike,
'Anchors': Anchors,
'RegressBoxes': RegressBoxes,
'PriorProbability': PriorProbability,
'ClipBoxes': ClipBoxes,
'FilterDetections': FilterDetections,
}
with custom_object_scope(custom_objects):
backbone = models.backbone('resnet50')
model = backbone.retinanet(500)
prediction_model = retinanet_bbox(model=model)
# prediction_model.load_weights("...your weights here...")
return prediction_model, custom_objects
def create_models(backbone_retinanet,
num_classes,
weights,
multi_gpu=0,
freeze_backbone=False):
""" Creates three models (model, training_model, prediction_model).
Args
backbone_retinanet : A function to call to create a retinanet model with a given backbone.
num_classes : The number of classes to train.
weights : The weights to load into the model.
multi_gpu : The number of GPUs to use for training.
freeze_backbone : If True, disables learning for the backbone.
Returns
model : The base model. This is also the model that is saved in snapshots.
training_model : The training model. If multi_gpu=0, this is identical to model.
prediction_model : The model wrapped with utility functions to perform object detection (applies regression values and performs NMS).
"""
modifier = freeze_model if freeze_backbone else None
# Keras recommends initialising a multi-gpu model on the CPU to ease weight sharing, and to prevent OOM errors.
# optionally wrap in a parallel model
if multi_gpu > 1:
from keras.utils import multi_gpu_model
with tf.device('/cpu:0'):
model = model_with_weights(backbone_retinanet(num_classes,
modifier=modifier),
weights=weights,
skip_mismatch=True)
training_model = multi_gpu_model(model, gpus=multi_gpu)
else:
model = model_with_weights(backbone_retinanet(num_classes,
modifier=modifier),
weights=weights,
skip_mismatch=True)
training_model = model
# make prediction model
prediction_model = retinanet_bbox(model=model)
# compile model
training_model.compile(
loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.sgd(lr=1e-2,
momentum=0.9,
decay=.0001,
nesterov=True,
clipnorm=1) #, clipnorm=0.001)
)
return model, training_model, prediction_model
def main(args=None):
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
# create object that stores backbone information
backbone = models.backbone(args.backbone)
# optionally load config parameters
if args.config:
args.config = read_config_file(args.config)
# create the generators
train_generator = create_generators(args, backbone.preprocess_image)
# create the model
if args.snapshot is not None:
print('Loading model')
model = models.load_model(args.snapshot, backbone_name=args.backbone)
training_model = model
anchor_params = None
if args.config and 'anchor_parameters' in args.config:
anchor_params = parse_anchor_parameters(args.config)
prediction_model = retinanet_bbox(model=model,
anchor_params=anchor_params)
else:
weights = args.weights
if weights is None and args.imagenet_weights:
weights = backbone.download_imagenet()
print('Creating model')
model, training_model, prediction_model = create_models(
backbone_retinanet=backbone.retinanet,
num_classes=train_generator.num_classes(),
weights=weights,
multi_gpu=args.multi_gpu,
freeze_backbone=args.freeze_backbone,
lr=args.lr,
config=args.config)
# this lets the generator compute backbone layer shapes using the actual backbone model
if 'densenet' in args.backbone:
train_generator.compute_shapes = make_shapes_callback(model)
# create the callbacks
callbacks = create_callbacks(model, training_model, prediction_model, args)
# start training
return training_model.fit_generator(
generator=train_generator,
steps_per_epoch=train_generator.size() // args.batch_size,
epochs=args.epochs,
verbose=1,
callbacks=callbacks)
def create_models(backbone_retinanet, num_classes, weights, multi_gpu=0,
freeze_backbone=False, lr=1e-5, config=None, regularisation=3.0):
""" Creates three models (model, training_model, prediction_model).
Args
backbone_retinanet : A function to call to create a retinanet model with a given backbone.
num_classes : The number of classes to train.
weights : The weights to load into the model.
multi_gpu : The number of GPUs to use for training.
freeze_backbone : If True, disables learning for the backbone.
config : Config parameters, None indicates the default configuration.
Returns
model : The base model. This is also the model that is saved in snapshots.
training_model : The training model. If multi_gpu=0, this is identical to model.
prediction_model : The model wrapped with utility functions to perform object detection (applies regression values and performs NMS).
"""
modifier = freeze_model if freeze_backbone else None
# load anchor parameters, or pass None (so that defaults will be used)
anchor_params = None
num_anchors = None
if config and 'anchor_parameters' in config:
anchor_params = parse_anchor_parameters(config)
num_anchors = anchor_params.num_anchors()
# Keras recommends initialising a multi-gpu model on the CPU to ease weight sharing, and to prevent OOM errors.
# optionally wrap in a parallel model
if multi_gpu > 1:
from keras.utils import multi_gpu_model
with tf.device('/cpu:0'):
model = model_with_weights(backbone_retinanet(num_classes, num_anchors=num_anchors, modifier=modifier), weights=weights, skip_mismatch=True)
training_model = multi_gpu_model(model, gpus=multi_gpu)
else:
model = model_with_weights(backbone_retinanet(num_classes, num_anchors=num_anchors, modifier=modifier), weights=weights, skip_mismatch=True)
training_model = model
# make prediction model
prediction_model = retinanet_bbox(model=model, anchor_params=anchor_params)
# compile model
if regularisation != 3.0:
print(f"regularisation = {regularisation}")
training_model.compile(
loss={
'regression' : losses.smooth_l1(regularisation),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=lr, clipnorm=0.001)
)
return model, training_model, prediction_model
def create_prediction_models(self):
"""
Create inference models
:return: The inference model
"""
return_models = {}
if self.vision_model_config['name'] == 'retinanet':
return_models['vision'] = retinanet_bbox(model=self.vision_model, anchor_params=self.vision_model_config['anchor_params'])
if self.language_translation_model_config['name'] == 'transformer':
return_models['language_translation'] = transformer_inference(model=self.language_translation_model)
return_models['fusionnet'] = self.fusionnet_model
return return_models
def main(args=None):
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
# make sure keras is the minimum required version
check_keras_version()
# optionally choose specific GPU
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
keras.backend.tensorflow_backend.set_session(get_session())
# make save path if it doesn't exist
if args.save_path is not None and not os.path.exists(args.save_path):
os.makedirs(args.save_path)
# create the generator
generator = create_generator(args)
# load the model
print('Loading model, this may take a second...')
# model = keras.models.load_model(args.model, custom_objects=custom_objects)
# shafin
inputs = keras.layers.Input(shape=(None, None, 3))
resnet = keras_resnet.models.ResNet50(inputs,
include_top=False,
freeze_bn=True)
model = retinanet.retinanet_bbox(inputs=inputs,
num_classes=80,
backbone=resnet)
model.load_weights(args.model)
# print model summary
print(model.summary())
# start evaluation
average_precisions = evaluate(generator,
model,
iou_threshold=args.iou_threshold,
score_threshold=args.score_threshold,
max_detections=args.max_detections,
save_path=args.save_path)
# print evaluation
for label, average_precision in average_precisions.items():
print(generator.label_to_name(label),
'{:.4f}'.format(average_precision))
print('mAP: {:.4f}'.format(
sum(average_precisions.values()) / len(average_precisions)))
def train(self,
annotation_path,
class_path,
snapshot_src=None,
tensorboard_dir=None,
snapshot_dir=None,
snapshot_tag=None,
snapshot_freq=10):
""" Train routine.
Params:
annotation_path: file path for target annotations
class_path: file path for class-id mapping
snapshot_src: file path of snapshot to continue training
tensorboard_dir: directory for saving tensorboard
snapshot_dir: directory for saving snapshots
shapshot_tag: tag used as filename identifier for snapshots
shapshot_freq: epoch interval for saving snapshots
"""
if self._epoch % snapshot_freq != 0:
print(
"Error: Snapshot saving interval should be set to factor of total epochns ({})."
.format(self._epoch))
return
# Load network backbone
generator = self.__create_generator(annotation_path, class_path)
if snapshot_src:
model = models.load_model(snapshot_src,
backbone_name=self._backbone_name)
training_model = model
prediction_model = retinanet_bbox(
model=model, anchor_params=self._anchor_params)
else:
model, training_model, prediction_model = self.__create_models(
self._backbone.retinanet, generator.num_classes())
print(model.summary())
if 'vgg' in self._backbone_name or 'densenet' in self._backbone_name:
generator.compute_shapes = make_shapes_callback(model)
callbacks = self.__create_callbacks(model, training_model,
prediction_model, tensorboard_dir,
snapshot_dir, snapshot_tag,
snapshot_freq)
training_model.fit_generator(generator=generator,
steps_per_epoch=self._step,
epochs=self._epoch,
verbose=1,
callbacks=callbacks)
def get_test_model(weights, num_classes):
"""Returns an inference retinanet model.
Args:
weights: Initial weights.
num_classes: Number of classes to detect.
n_gpu: Number of gpu, if above 1, will set up a multi gpu model.
Returns:
The inference model.
"""
model = get_model(weights, num_classes, freeze=True, n_gpu=1)[0]
test_model = retinanet_bbox(model=model)
return test_model
def __create_models(self, backbone_retinanet, num_classes):
anchor_params = self._anchor_params
num_anchors = anchor_params.num_anchors()
model = backbone_retinanet(num_classes,
num_anchors=num_anchors,
modifier=None)
training_model = model
prediction_model = retinanet_bbox(model=model,
anchor_params=anchor_params)
training_model.compile(loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=1e-5,
clipnorm=0.001))
return model, training_model, prediction_model
def main(args=None):
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
# make sure keras is the minimum required version
check_keras_version()
# optionally choose specific GPU
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
keras.backend.tensorflow_backend.set_session(get_session())
# optionally load config parameters
if args.config:
args.config = read_config_file(args.config)
# create the generators
generator = create_generators(args)
# create the model
print('Loading model, this may take a second...')
model = models.load_model(args.snapshot, backbone_name=args.backbone)
anchor_params = None
if args.config and 'anchor_parameters' in args.config:
anchor_params = parse_anchor_parameters(args.config)
prediction_model = retinanet_bbox(model=model, anchor_params=anchor_params)
# print model summary
prediction_model.summary()
# this lets the generator compute backbone layer shapes using the actual backbone model
if 'vgg' in args.backbone or 'densenet' in args.backbone:
generator.compute_shapes = make_shapes_callback(model)
# inference
boxes, scores, labels = prediction_model.predict_generator(
generator=generator,
verbose=1,
)
# generate COCO annotations
create_coco_annotations(boxes, scores, labels, generator.image_ids)
'image_max_side': image_max_side,
'preprocess_image': backbone_model.preprocess_image,
}
train_generator = CSVGenerator(annotations_path, classes_path, **common_args)
if validations_path != '':
validation_generator = CSVGenerator(validations_path, classes_path,
**common_args)
else:
validation_generator = None
if snapshot is not None:
print('Loading model, this may take a second...')
model = models.load_model(snapshot, backbone_name=backbone)
training_model = model
prediction_model = retinanet_bbox(model=model)
else:
weights = weights
# default to imagenet if nothing else is specified
if weights is None and imagenet_weights:
weights = backbone_model.download_imagenet()
print('Creating model, this may take a second...')
modifier = freeze_model if freeze_backbone else None
model = backbone_model.retinanet(train_generator.num_classes(),
modifier=modifier)
if weights is not None:
model.load_weights(weights, by_name=True, skip_mismatch=True)
training_model = model
def retinanet_mask(
inputs,
num_classes,
nms=True,
name='retinanet-mask',
roi_submodels=None,
**kwargs
):
""" Construct a RetinaNet mask model on top of a retinanet bbox model.
This model uses the retinanet bbox model and appends a few layers to compute masks.
# Arguments
inputs : List of keras.layers.Input. The first input is the image, the second input the blob of masks.
num_classes : Number of classes to classify.
name : Name of the model.
*kwargs : Additional kwargs to pass to the retinanet bbox model.
# Returns
Model with inputs as input and as output the output of each submodel for each pyramid level and the detections.
The order is as defined in submodels. Using default values the output is:
```
[
regression, classification, boxes_masks, boxes, masks
]
```
"""
if roi_submodels is None:
roi_submodels = default_roi_submodels(num_classes)
image = inputs
image_shape = Shape()(image)
bbox_model = retinanet.retinanet_bbox(inputs=image, num_classes=num_classes, nms=False, **kwargs)
# parse outputs
regression = bbox_model.outputs[0]
classification = bbox_model.outputs[1]
other = bbox_model.outputs[2:-1]
boxes = bbox_model.outputs[-1]
fpn = [bbox_model.get_layer(name).output for name in ['P3', 'P4', 'P5', 'P6', 'P7']]
# get the region of interest features
top_boxes, top_classification, rois = RoiAlign()([image_shape, boxes, classification] + fpn)
# optionally apply non maximum suppression
if nms:
top_classification = NonMaximumSuppression(name='nms')([top_boxes, top_classification])
# estimate masks
# TODO: Change this so that it iterates over roi_submodels
masks = roi_submodels[0][1](rois)
# concatenate boxes and masks together
boxes_masks = ConcatenateBoxesMasks(name='boxes_masks')([top_boxes, masks])
# reconstruct the new output
outputs = [regression, classification] + other + [boxes_masks, top_boxes, top_classification, masks]
# construct the model
return keras.models.Model(inputs=inputs, outputs=outputs, name=name)
def main(args=None):
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
# create object that stores backbone information
backbone = models.backbone(args.backbone)
# make sure keras is the minimum required version
check_keras_version()
# optionally choose specific GPU
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
keras.backend.set_session(get_session())
# optionally load config parameters
if args.config:
args.config = read_config_file(args.config)
# create the generators
train_generator, validation_generator = create_generators(
args, backbone.preprocess_image)
# create the model
if args.snapshot is not None:
print('Loading model, this may take a second...')
model = models.load_model(args.snapshot, backbone_name=args.backbone)
training_model = model
anchor_params = None
if args.config and 'anchor_parameters' in args.config:
anchor_params = parse_anchor_parameters(args.config)
prediction_model = retinanet_bbox(model=model,
anchor_params=anchor_params)
else:
weights = args.weights
# default to imagenet if nothing else is specified
if weights is None and args.imagenet_weights:
weights = backbone.download_imagenet()
print('Creating model, this may take a second...')
model, training_model, prediction_model = create_models(
backbone_retinanet=backbone.retinanet,
num_classes=train_generator.num_classes(),
weights=weights,
multi_gpu=args.multi_gpu,
freeze_backbone=args.freeze_backbone,
lr=args.lr,
config=args.config)
# print model summary
print(model.summary())
# this lets the generator compute backbone layer shapes using the actual backbone model
# if 'vgg' in args.backbone or 'densenet' in args.backbone:
# train_generator.compute_shapes = make_shapes_callback(model)
# if validation_generator:
# validation_generator.compute_shapes = train_generator.compute_shapes
# create the callbacks
callbacks = create_callbacks(model,
training_model,
prediction_model,
validation_generator,
args,
eval_batch_size=args.val_batch_size)
# Use multiprocessing if workers > 0
if args.workers > 0:
use_multiprocessing = True
else:
use_multiprocessing = False
# start training
training_model.fit_generator(generator=train_generator,
steps_per_epoch=args.steps,
epochs=args.epochs,
verbose=1,
callbacks=callbacks,
workers=args.workers,
use_multiprocessing=use_multiprocessing,
max_queue_size=args.max_queue_size)
def main(args=None):
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
# create object that stores backbone information
backbone = models.backbone(args.backbone)
# make sure keras is the minimum required version
check_keras_version()
# optionally choose specific GPU
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
keras.backend.tensorflow_backend.set_session(get_session())
# create the generators
train_generator, validation_generator = create_generators(
args, backbone.preprocess_image)
# create the model
if args.snapshot is not None:
print('Loading model, this may take a second...')
model = models.load_model(args.snapshot, backbone_name=args.backbone)
training_model = model
prediction_model = retinanet_bbox(model=model)
else:
weights = args.weights
# default to imagenet if nothing else is specified
if weights is None and args.imagenet_weights:
weights = backbone.download_imagenet()
else:
print("Don't load weight!")
print('Creating model, this may take a second...')
model, training_model, prediction_model = create_models(
backbone_retinanet=backbone.retinanet,
num_classes=train_generator.num_classes(),
weights=weights,
multi_gpu=args.multi_gpu,
freeze_backbone=args.freeze_backbone)
# print model summary
print(model.summary())
# create the callbacks
callbacks = create_callbacks(
model,
training_model,
prediction_model,
validation_generator,
args,
)
# start training
training_model.fit_generator(generator=train_generator,
steps_per_epoch=args.steps,
epochs=args.epochs,
verbose=1,
callbacks=callbacks,
validation_data=validation_generator,
validation_steps=20)
def main(args=None):
global config
from keras import backend as K
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
print('Arguments: {}'.format(args))
# create object that stores backbone information
backbone = models.backbone(args.backbone)
# make sure keras is the minimum required version
check_keras_version()
# optionally choose specific GPU
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
keras.backend.tensorflow_backend.set_session(get_session())
# create the generators
train_generator, validation_generator = create_generators(
args, backbone.preprocess_image)
# create the model
if args.snapshot is not None:
print('Loading model, this may take a second...')
model = models.load_model(args.snapshot, backbone_name=args.backbone)
training_model = model
anchor_params = None
if 'anchor_parameters' in config:
anchor_params = parse_anchor_parameters(config)
prediction_model = retinanet_bbox(model=model,
anchor_params=anchor_params)
else:
weights = args.weights
# default to imagenet if nothing else is specified
if weights is None and args.imagenet_weights:
weights = backbone.download_imagenet()
print('Creating model, this may take a second...')
model, training_model, prediction_model = create_models(
backbone_retinanet=backbone.retinanet,
num_classes=train_generator.num_classes(),
weights=weights,
args=args,
multi_gpu=args.multi_gpu,
freeze_backbone=args.freeze_backbone,
config=config)
# print model summary
print(model.summary())
print('Learning rate: {}'.format(K.get_value(model.optimizer.lr)))
if args.lr > 0.0:
K.set_value(model.optimizer.lr, args.lr)
print('Updated learning rate: {}'.format(
K.get_value(model.optimizer.lr)))
# this lets the generator compute backbone layer shapes using the actual backbone model
if 'vgg' in args.backbone or 'densenet' in args.backbone:
train_generator.compute_shapes = make_shapes_callback(model)
if validation_generator:
validation_generator.compute_shapes = train_generator.compute_shapes
# create the callbacks
callbacks = create_callbacks(
model,
training_model,
prediction_model,
validation_generator,
args,
)
init_epoch = 0
try:
if args.snapshot:
init_epoch = int(args.snapshot.split("_")[-2])
except:
pass
# init_epoch = 6
print('Init epoch: {}'.format(init_epoch))
# start training
training_model.fit_generator(
generator=train_generator,
steps_per_epoch=args.steps,
epochs=args.epochs,
verbose=1,
callbacks=callbacks,
initial_epoch=init_epoch,
)
def main(args=None):
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
# create object that stores backbone information
backbone = models.backbone(args.backbone)
# make sure keras is the minimum required version
check_keras_version()
# optionally choose specific GPU
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
keras.backend.tensorflow_backend.set_session(get_session())
# create the generators
train_generator, validation_generator = create_generators(args, backbone.preprocess_image)
# create the model
if args.snapshot is not None:
print('Loading model, this may take a second...')
model = models.load_model(args.snapshot, backbone_name=args.backbone)
training_model = model
prediction_model = retinanet_bbox(model=model)
else:
weights = args.weights
# default to imagenet if nothing else is specified
if weights is None and args.imagenet_weights:
weights = backbone.download_imagenet()
print('Creating model, this may take a second...')
model, training_model, prediction_model = create_models(
backbone_retinanet=backbone.retinanet,
num_classes=train_generator.num_classes(),
weights=weights,
multi_gpu=args.multi_gpu,
freeze_backbone=args.freeze_backbone
)
# print model summary
print(model.summary())
# this lets the generator compute backbone layer shapes using the actual backbone model
if 'vgg' in args.backbone or 'densenet' in args.backbone:
train_generator.compute_shapes = make_shapes_callback(model)
if validation_generator:
validation_generator.compute_shapes = train_generator.compute_shapes
# create the callbacks
callbacks = create_callbacks(
model,
training_model,
prediction_model,
validation_generator,
args,
)
# start training
history=training_model.fit_generator(
generator=train_generator,
steps_per_epoch=args.steps,
epochs=args.epochs,
verbose=1,
callbacks=callbacks,
)
timestr=time.strftime("%Y-%m-%d-%H%M")
history_path=os.path.join(
args.snapshot_path,
'{timestr}_{backbone}.csv'.format(timestr=timestr,backbone=args.backbone, dataset_type=args.dataset_type)
)
pd.DataFrame(history.history).to_csv(history_path)
def main(args=None, data=None, DeepForest_config=None, experiment=None):
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
# create object that stores backbone information
backbone = models.backbone(args.backbone)
# make sure keras is the minimum required version
check_keras_version()
# optionally choose specific GPU
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
keras.backend.tensorflow_backend.set_session(get_session())
# create the generators
train_generator, validation_generator = create_generators(
args, data, DeepForest_config=DeepForest_config)
# create the model
if args.snapshot is not None:
print('Loading model, this may take a secondkeras-retinanet.\n')
model = models.load_model(args.snapshot, backbone_name=args.backbone)
training_model = model
prediction_model = retinanet_bbox(
model=model, nms_threshold=DeepForest_config["nms_threshold"])
else:
weights = args.weights
# default to imagenet if nothing else is specified
if weights is None and args.imagenet_weights:
weights = backbone.download_imagenet()
print('Creating model, this may take a secondkeras-retinanet .')
model, training_model, prediction_model = create_models(
backbone_retinanet=backbone.retinanet,
num_classes=train_generator.num_classes(),
weights=weights,
multi_gpu=args.multi_gpu,
freeze_backbone=args.freeze_backbone,
nms_threshold=DeepForest_config["nms_threshold"])
# print model summary
#print(model.summary())
# this lets the generator compute backbone layer shapes using the actual backbone model
if 'vgg' in args.backbone or 'densenet' in args.backbone:
compute_anchor_targets = functools.partial(
anchor_targets_bbox, shapes_callback=make_shapes_callback(model))
train_generator.compute_anchor_targets = compute_anchor_targets
if validation_generator is not None:
validation_generator.compute_anchor_targets = compute_anchor_targets
matched = []
for entry in validation_generator.image_data.values():
test = entry in train_generator.image_data.values()
matched.append(test)
if sum(matched) > 0:
raise Exception(
"%.2f percent of validation windows are in training data" %
(100 * sum(matched) / train_generator.size()))
else:
print("Test passed: No overlapping data in training and validation")
#start training
cp.enable()
training_model.fit_generator(generator=train_generator,
steps_per_epoch=train_generator.size() /
DeepForest_config["batch_size"],
epochs=args.epochs,
verbose=1,
shuffle=False,
callbacks=None,
workers=DeepForest_config["workers"],
use_multiprocessing=True)
cp.disable()
def create_models(backbone_retinanet,
num_classes,
weights,
multi_gpu=0,
freeze_backbone=False,
lr=1e-5,
config=None,
targets=None,
freeze_layers=0,
modifier=None):
"""Creates three models (model, training_model, prediction_model).
Args:
backbone_retinanet : A function to call to create a retinanet model
with a given backbone.
num_classes : The number of classes to train.
weights : The weights to load into the model.
multi_gpu : The number of GPUs to use for training.
freeze_backbone : If True, disables learning for the backbone.
config : Config parameters, None indicates the default configuration.
targets : Target tensors if training a model with tfrecord inputs
freeze_layers : int layer number to freeze from bottom of the retinanet
network during finetuning. e.g. 10 will set
layers 0:10 to layer.trainable = False. 0 is default, no freezing.
modifier : function that takes in a model and freezes resnet layers,
returns modified object
Returns:
model : The base model.
This is also the model that is saved in snapshots.
training_model : The training model.
If multi_gpu=0, this is identical to model.
prediction_model : The model wrapped with utility functions to perform
object detection (applies regression values and performs NMS).
"""
# if not modifier:
modifier = freeze_model if freeze_backbone else None
# load anchor parameters, or pass None (so that defaults will be used)
anchor_params = None
num_anchors = None
if config and 'anchor_parameters' in config:
anchor_params = parse_anchor_parameters(config)
num_anchors = anchor_params.num_anchors()
# Keras recommends initialising a multi-gpu model on the CPU to ease weight sharing,
# and to prevent OOM errors.
# optionally wrap in a parallel model
if multi_gpu > 1:
from keras.utils import multi_gpu_model
with tf.device('/cpu:0'):
model = model_with_weights(backbone_retinanet(
num_classes, num_anchors=num_anchors, modifier=modifier),
weights=weights,
skip_mismatch=True)
training_model = multi_gpu_model(model, gpus=multi_gpu)
else:
model = model_with_weights(backbone_retinanet(num_classes,
num_anchors=num_anchors,
modifier=modifier),
weights=weights,
skip_mismatch=True)
training_model = model
# make prediction model
prediction_model = retinanet_bbox(model=model, anchor_params=anchor_params)
# Compile model
if targets:
# tfdataset target tensor from tfrecords pipelione
training_model.compile(loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=lr,
clipnorm=0.001),
target_tensors=targets)
else:
# compile model
training_model.compile(loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=lr,
clipnorm=0.001))
return model, training_model, prediction_model
def main(args=None):
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
# make sure keras is the minimum required version
check_keras_version()
# optionally choose specific GPU
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
keras.backend.tensorflow_backend.set_session(get_session())
# create the generators
train_generator, validation_generator = create_generators(args)
if 'resnet' in args.backbone:
from keras_retinanet.models import retinanet
from ..models.resnet_vocab_w2v import custom_objects
from keras_retinanet.models import retinanet_vocab_w2v
import keras_resnet.models
elif 'mobilenet' in args.backbone:
from ..models.mobilenet import mobilenet_retinanet as retinanet, custom_objects, download_imagenet
else:
raise NotImplementedError('Backbone \'{}\' not implemented.'.format(
args.backbone))
# create the model
if args.snapshot is not None:
print('Loading model, this may take a second...')
model = keras.models.load_model(args.snapshot,
custom_objects=custom_objects)
training_model = model
prediction_model = model
else:
inputs = keras.layers.Input(shape=(None, None, 3))
resnet = keras_resnet.models.ResNet50(inputs,
include_top=False,
freeze_bn=True)
training_model = retinanet_vocab_w2v.retinanet_bbox(inputs=inputs,
num_classes=65,
backbone=resnet)
# compile model
training_model.compile(loss={
'regression': losses.smooth_l1(),
'classification': losses.polar()
},
optimizer=keras.optimizers.adam(lr=1e-5,
clipnorm=0.001))
# Loading a Pre-trained Model
inputs = keras.layers.Input(shape=(None, None, 3))
cocomodel_resnet = keras_resnet.models.ResNet50(inputs,
include_top=False,
freeze_bn=True)
cocomodel65 = retinanet.retinanet_bbox(inputs=inputs,
num_classes=65,
backbone=cocomodel_resnet)
cocomodel65.load_weights('Model/resnet50_csv_50_focal_seen_w2v.h5')
for i in range(0, cocomodel65.layers.__len__(), 1):
wei = cocomodel65.layers[i].get_weights()
try:
training_model.layers[i].set_weights(wei)
except Exception as e:
print i
print 'Pre-trained load done'
print(training_model.summary())
# create the callbacks
callbacks = create_callbacks(
training_model,
training_model,
training_model,
validation_generator,
args,
)
# start training
training_model.fit_generator(
generator=train_generator,
steps_per_epoch=args.steps,
epochs=args.epochs,
verbose=1,
callbacks=callbacks,
)
def get_retinanet_predictions_for_files(files,
out_dir,
pretrained_model_path,
min_side,
max_side,
backbonename,
label_level,
show_debug_images=False):
backbone = models.backbone(backbonename)
print('Label Size:', len(label_level))
model = model_with_weights(backbone.retinanet(len(label_level),
modifier=None),
weights=pretrained_model_path,
skip_mismatch=False)
# make prediction model
prediction_model = retinanet_bbox(model=model)
print('Proc {} files...'.format(len(files)))
count = 0
start = time.time()
for f in files:
id = os.path.basename(f)[:-4]
cache_path = out_dir + id + '.pkl'
if os.path.isfile(cache_path):
continue
# preprocess image for network
image = read_image_bgr_fast(f)
if show_debug_images:
# copy to draw on
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
image = preprocess_image(image)
image, scale = resize_image(image,
min_side=min_side,
max_side=max_side)
image = np.expand_dims(image, axis=0)
# process image
boxes, scores, labels = prediction_model.predict_on_batch(image)
if count % 100 == 0:
print(
"Count: {}, Processing time: {:.2f} sec, Detections shape: {}".
format(count,
time.time() - start, boxes.shape))
count += 1
if show_debug_images:
boxes_init = boxes.copy()
boxes_init /= scale
boxes[:, :, 0] /= image.shape[2]
boxes[:, :, 2] /= image.shape[2]
boxes[:, :, 1] /= image.shape[1]
boxes[:, :, 3] /= image.shape[1]
if show_debug_images:
if len(boxes_init) > 0:
show_image_debug(label_level, draw.astype(np.uint8),
boxes_init[:1], scores[:1], labels[:1])
save_in_file_fast((boxes, scores, labels), cache_path)
def main(forest_object,
args=None,
input_type="fit_generator",
list_of_tfrecords=None,
comet_experiment=None):
"""
Main Training Loop
Args:
forest_object: a deepforest class object
args: Keras retinanet argparse
list_of_tfrecords: list of tfrecords to parse
input_type: "fit_generator" or "tfrecord" input type
"""
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
# create object that stores backbone information
backbone = models.backbone(args.backbone)
# make sure keras is the minimum required version
check_keras_version()
# optionally choose specific GPU
if args.gpu:
setup_gpu(args.gpu)
# optionally load config parameters
if args.config:
args.config = read_config_file(args.config)
# data input
if input_type == "fit_generator":
# create the generators
train_generator, validation_generator = create_generators(
args, backbone.preprocess_image)
# placeholder target tensor for creating models
targets = None
elif input_type == "tfrecord":
# Create tensorflow iterators
iterator = tfrecords.create_dataset(list_of_tfrecords, args.batch_size)
next_element = iterator.get_next()
# Split into inputs and targets
inputs = next_element[0]
targets = [next_element[1], next_element[2]]
validation_generator = None
else:
raise ValueError(
"{} input type is invalid. Only 'tfrecord' or 'for_generator' "
"input types are accepted for model training".format(input_type))
# create the model
if args.snapshot is not None:
print('Loading model, this may take a second...')
model = models.load_model(args.snapshot, backbone_name=args.backbone)
training_model = model
anchor_params = None
if args.config and 'anchor_parameters' in args.config:
anchor_params = parse_anchor_parameters(args.config)
prediction_model = retinanet_bbox(model=model,
anchor_params=anchor_params)
else:
weights = args.weights
# default to imagenet if nothing else is specified
if weights is None and args.imagenet_weights:
weights = backbone.download_imagenet()
print('Creating model, this may take a second...')
if input_type == "fit_generator":
num_of_classes = train_generator.num_classes()
else:
# Add background class
num_of_classes = len(forest_object.labels.keys())
model, training_model, prediction_model = create_models(
backbone_retinanet=backbone.retinanet,
num_classes=num_of_classes,
weights=weights,
multi_gpu=args.multi_gpu,
freeze_backbone=args.freeze_backbone,
lr=args.lr,
config=args.config,
targets=targets,
freeze_layers=args.freeze_layers)
# print model summary
print(model.summary())
# this lets the generator compute backbone layer shapes using the actual backbone model
if 'vgg' in args.backbone or 'densenet' in args.backbone:
train_generator.compute_shapes = make_shapes_callback(model)
if validation_generator:
validation_generator.compute_shapes = train_generator.compute_shapes
# create the callbacks
callbacks = create_callbacks(model, training_model, prediction_model,
validation_generator, args, comet_experiment)
if not args.compute_val_loss:
validation_generator = None
# start training
if input_type == "fit_generator":
history = training_model.fit_generator(
generator=train_generator,
steps_per_epoch=args.steps,
epochs=args.epochs,
verbose=1,
callbacks=callbacks,
workers=args.workers,
use_multiprocessing=args.multiprocessing,
max_queue_size=args.max_queue_size,
validation_data=validation_generator)
elif input_type == "tfrecord":
# Fit model
history = training_model.fit(x=inputs,
steps_per_epoch=args.steps,
epochs=args.epochs,
callbacks=callbacks)
else:
raise ValueError(
"{} input type is invalid. Only 'tfrecord' or 'for_generator' "
"input types are accepted for model training".format(input_type))
# Assign history to deepforest model class
forest_object.history = history
# return trained model
return model, prediction_model, training_model
def train_main(args=None):
# parse arguments
if args is None:
args = sys.argv[1:]
args = train.parse_args(args)
# create object that stores backbone information
backbone = models.backbone(args.backbone)
# make sure keras is the minimum required version
train.check_keras_version()
# optionally choose specific GPU
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
keras.backend.tensorflow_backend.set_session(train.get_session())
# create the generators
train_generator, validation_generator = train.create_generators(args)
# create the model
if args.snapshot is not None:
print('Loading model, this may take a second...')
model = models.load_model(args.snapshot, backbone_name=args.backbone)
training_model = model
prediction_model = retinanet_bbox(model=model)
else:
weights = args.weights
# default to imagenet if nothing else is specified
if weights is None and args.imagenet_weights:
weights = backbone.download_imagenet()
print('Creating model, this may take a second...')
model, training_model, prediction_model = train.create_models(
backbone_retinanet=backbone.retinanet,
num_classes=train_generator.num_classes(),
weights=weights,
multi_gpu=args.multi_gpu,
freeze_backbone=args.freeze_backbone)
# print model summary
# print(model.summary())
# this lets the generator compute backbone layer shapes using the actual backbone model
if 'vgg' in args.backbone or 'densenet' in args.backbone:
compute_anchor_targets = functools.partial(
anchor_targets_bbox, shapes_callback=make_shapes_callback(model))
train_generator.compute_anchor_targets = compute_anchor_targets
if validation_generator is not None:
validation_generator.compute_anchor_targets = compute_anchor_targets
# create the callbacks
callbacks = train.create_callbacks(
model,
training_model,
prediction_model,
validation_generator,
args,
)
callbacks = add_classification_callbacks(args, callbacks, None)
# start training
training_model.fit_generator(generator=train_generator,
steps_per_epoch=args.steps,
epochs=args.epochs,
verbose=1,
callbacks=callbacks,
initial_epoch=args.initial_epoch)
# cleanup at the end of every epoch since we keep loading prediction models
del training_model
del model
del prediction_model
K.clear_session()
def main(args=None):
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
# create object that stores backbone information
backbone = models.backbone(args.backbone)
# make sure keras and tensorflow are the minimum required version
check_keras_version()
check_tf_version()
# optionally choose specific GPU
if args.gpu is not None:
setup_gpu(args.gpu)
# optionally load config parameters
if args.config:
args.config = read_config_file(args.config)
# create the generators
train_generator, validation_generator = create_generators(args, backbone.preprocess_image)
# create the model
if args.snapshot is not None:
print('Loading model, this may take a second...')
model = models.load_model(args.snapshot, backbone_name=args.backbone)
training_model = model
anchor_params = None
if args.config and 'anchor_parameters' in args.config:
anchor_params = parse_anchor_parameters(args.config)
prediction_model = retinanet_bbox(model=model, anchor_params=anchor_params)
else:
weights = args.weights
# default to imagenet if nothing else is specified
if weights is None and args.imagenet_weights:
weights = backbone.download_imagenet()
print('Creating model, this may take a second...')
model, training_model, prediction_model = create_models(
backbone_retinanet=backbone.retinanet,
num_classes=train_generator.num_classes(),
weights=weights,
multi_gpu=args.multi_gpu,
freeze_backbone=args.freeze_backbone,
lr=args.lr,
config=args.config,
regularisation=args.regularisation
)
# print model summary
#print(model.summary())
# this lets the generator compute backbone layer shapes using the actual backbone model
if 'vgg' in args.backbone or 'densenet' in args.backbone:
train_generator.compute_shapes = make_shapes_callback(model)
if validation_generator:
validation_generator.compute_shapes = train_generator.compute_shapes
# create the callbacks
callbacks = create_callbacks(
model,
training_model,
prediction_model,
validation_generator,
args,
)
if not args.compute_val_loss:
validation_generator = None
# start training
return training_model.fit_generator(
generator=train_generator,
steps_per_epoch=args.steps,
epochs=args.epochs,
verbose=1,
callbacks=callbacks,
workers=args.workers,
use_multiprocessing=args.multiprocessing,
max_queue_size=args.max_queue_size,
validation_data=validation_generator,
initial_epoch=args.initial_epoch
)
def main(args=None):
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
# create object that stores backbone information
backbone = get_backbone(args.backbone)
# make sure keras is the minimum required version
check_keras_version()
if args.dataset_type == 'run':
model = load_model(args.model_path,
backbone_name=args.backbone,
convert=True)
labels_to_names = {0: 'cell'}
makedirs(args.save_path)
test_imlist = next(os.walk(args.run_path))[2]
for testimgcnt, img_path in enumerate(test_imlist):
image = get_image(img_path)
draw2 = get_image(img_path)
draw2 = draw2 / np.max(draw2)
# copy to draw on
# preprocess image for network
image = preprocess_image(image)
image, scale = resize_image(image)
print(scale)
# process image
start = time.time()
boxes, scores, labels = model.predict_on_batch(
np.expand_dims(image, axis=0))
print('processing time: ', time.time() - start)
# correct for image scale
boxes /= scale
# visualize detections
for box, score, label in zip(boxes[0], scores[0], labels[0]):
# scores are sorted so we can break
if score < 0.5:
break
color = label_color(label)
color = [255, 0, 255]
b = box.astype(int)
draw_box(draw2, b, color=color)
caption = '{} {:.3f}'.format(labels_to_names[label], score)
draw_caption(draw2, b, caption)
plt.imsave(
os.path.join(args.save_path,
'retinanet_output_' + str(testimgcnt)), draw2)
# optionally choose specific GPU
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
keras.backend.tensorflow_backend.set_session(get_session())
# create the generators
train_generator, validation_generator = create_generators(
args, backbone.preprocess_image)
# create the model
if args.snapshot is not None:
print('Loading model, this may take a second...')
model = load_model(args.snapshot, backbone_name=args.backbone)
training_model = model
prediction_model = retinanet_bbox(model=model)
else:
weights = args.weights
# default to imagenet if nothing else is specified
if weights is None and args.imagenet_weights:
weights = backbone.download_imagenet()
print('Creating model, this may take a second...')
model, training_model, prediction_model = create_models(
backbone_retinanet=backbone.retinanet,
num_classes=train_generator.num_classes(),
weights=weights,
multi_gpu=args.multi_gpu,
freeze_backbone=args.freeze_backbone)
print(model.summary())
# this lets the generator compute backbone layer shapes using the actual backbone model
if any(x in args.backbone for x in ['vgg', 'densenet', 'deepcell']):
train_generator.compute_shapes = make_shapes_callback(model)
if validation_generator:
validation_generator.compute_shapes = train_generator.compute_shapes
# create the callbacks
callbacks = create_callbacks(
model,
training_model,
prediction_model,
validation_generator,
args,
)
# start training
training_model.fit_generator(
generator=train_generator,
steps_per_epoch=args.steps,
epochs=args.epochs,
verbose=1,
callbacks=callbacks,
)
def main(args=None):
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
# create object that stores backbone information
backbone = models.backbone(args.backbone)
# make sure keras is the minimum required version
check_keras_version()
# optionally choose specific GPU
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
keras.backend.tensorflow_backend.set_session(get_session())
if args.anchors:
anchors_dict = get_anchors_params(args.anchors)
elif args.snapshot:
#search the anchors parameters configure beside models
anchors_path = os.path.join(args.snapshot,"anchors.yaml")
anchors_path = anchors_path if os.path.exists(anchors_path) else None
anchors_dict = get_anchors_params(anchors_path)
else:
#default anchors params
anchors_dict = get_anchors_params(None)
anchors_params = AnchorParameters(**anchors_dict)
if args.snapshots and args.anchors:
#save anchors configure beside models
makedirs(args.snapshot_path)
anchors_out = os.path.join(
args.snapshot_path,
"anchors.yaml"
)
save_anchors_params(anchors_dict,anchors_out)
# create the generators
train_generator, validation_generator = create_generators(args, backbone.preprocess_image)
# create the model
if args.snapshot is not None:
print('Loading model, this may take a second...')
model = models.load_model(args.snapshot, backbone_name=args.backbone)
training_model = model
prediction_model = retinanet_bbox(model=model,anchor_parameters = anchors_params)
else:
weights = args.weights
# default to imagenet if nothing else is specified
if weights is None and args.imagenet_weights:
weights = backbone.download_imagenet()
print('Creating model, this may take a second...')
model, training_model, prediction_model = create_models(
backbone_retinanet=backbone.retinanet,
num_classes=train_generator.num_classes(),
weights=weights,
multi_gpu=args.multi_gpu,
freeze_backbone=args.freeze_backbone,
anchors_path=args.anchors
)
# print model summary
print(model.summary())
# this lets the generator compute backbone layer shapes using the actual backbone model
if 'vgg' in args.backbone or 'densenet' in args.backbone:
train_generator.compute_shapes = make_shapes_callback(model)
if validation_generator:
validation_generator.compute_shapes = train_generator.compute_shapes
# create the callbacks
callbacks = create_callbacks(
model,
training_model,
prediction_model,
validation_generator,
args,
)
# start training
training_model.fit_generator(
generator=train_generator,
steps_per_epoch=args.steps,
epochs=args.epochs,
verbose=1,
callbacks=callbacks,
)
def main(args=None, data=None, DeepForest_config=None, experiment=None):
# parse arguments
print("parsing arguments")
if args is None:
args = sys.argv[1:]
args = parse_args(args)
# create object that stores backbone information
backbone = models.backbone(args.backbone)
# make sure keras is the minimum required version
print("Get keras version")
check_keras_version()
# optionally choose specific GPU
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
print("Get session")
keras.backend.tensorflow_backend.set_session(get_session())
# create the generators
print("Creating generators")
train_generator, validation_generator = create_generators(args, data, DeepForest_config=DeepForest_config)
#Log number of trees trained on
if experiment:
experiment.log_parameter("Number of Training Trees", train_generator.total_trees)
# create the model
if args.snapshot is not None:
print('Loading model, this may take a secondkeras-retinanet.\n')
model = models.load_model(args.snapshot, backbone_name=args.backbone)
training_model = model
prediction_model = retinanet_bbox(model=model, nms_threshold=DeepForest_config["nms_threshold"])
else:
weights = args.weights
# default to imagenet if nothing else is specified
if weights is None and args.imagenet_weights:
print("Loading imagenet weights")
weights = backbone.download_imagenet()
print('Creating model, this may take a secondkeras-retinanet .')
model, training_model, prediction_model = create_models(
backbone_retinanet=backbone.retinanet,
num_classes=train_generator.num_classes(),
weights=weights,
multi_gpu=args.multi_gpu,
freeze_backbone=args.freeze_backbone,
nms_threshold=DeepForest_config["nms_threshold"],
input_channels=DeepForest_config["input_channels"]
)
# print model summary
#print(model.summary())
# this lets the generator compute backbone layer shapes using the actual backbone model
if 'vgg' in args.backbone or 'densenet' in args.backbone:
compute_anchor_targets = functools.partial(anchor_targets_bbox, shapes_callback=make_shapes_callback(model))
train_generator.compute_anchor_targets = compute_anchor_targets
if validation_generator is not None:
validation_generator.compute_anchor_targets = compute_anchor_targets
# create the callbacks
callbacks = create_callbacks(
model,
training_model,
prediction_model,
train_generator,
validation_generator,
args,
experiment,
DeepForest_config
)
#Make sure no overlapping data
if validation_generator:
matched=[]
for entry in validation_generator.image_data.values():
test = entry in train_generator.image_data.values()
matched.append(test)
if sum(matched) > 0:
raise Exception("%.2f percent of validation windows are in training data" % (100 * sum(matched)/train_generator.size()))
else:
print("Test passed: No overlapping data in training and validation")
#start training
history = training_model.fit_generator(
generator=train_generator,
steps_per_epoch=train_generator.size()/DeepForest_config["batch_size"],
epochs=args.epochs,
verbose=2,
shuffle=False,
callbacks=callbacks,
workers=DeepForest_config["workers"],
use_multiprocessing=DeepForest_config["use_multiprocessing"],
max_queue_size=DeepForest_config["max_queue_size"]
)
#return path snapshot of final epoch
saved_models = glob.glob(os.path.join(args.snapshot_path,"*.h5"))
saved_models.sort()
#Return model if found
if len(saved_models) > 0:
return saved_models[-1]
