def create_models(backbone_retinanet,
num_classes,
weights,
args,
num_gpus=0,
freeze_backbone=False,
lr=1e-5,
config=None):
"""
Creates three models (model, training_model, prediction_model).
Args
backbone_retinanet: A function to call to create a retinanet model with a given backbone.
num_classes: The number of classes to train.
weights: The weights to load into the model.
num_gpus: The number of GPUs to use for training.
freeze_backbone: If True, disables learning for the backbone.
config: Config parameters, None indicates the default configuration.
Returns
model: The base model. This is also the model that is saved in snapshots.
training_model: The training model. If num_gpus=0, this is identical to model.
prediction_model: The model wrapped with utility functions to perform object detection (applies regression values and performs NMS).
"""
modifier = freeze_model if freeze_backbone else None
# Keras recommends initialising a multi-gpu model on the CPU to ease weight sharing, and to prevent OOM errors.
# optionally wrap in a parallel model
if num_gpus > 1:
from keras.utils import multi_gpu_model
with tf.device('/cpu:0'):
model = model_with_weights(backbone_retinanet(num_classes,
modifier=modifier),
weights=weights,
skip_mismatch=True)
training_model = multi_gpu_model(model, gpus=num_gpus)
else:
model = model_with_weights(backbone_retinanet(num_classes,
modifier=modifier),
weights=weights,
skip_mismatch=True)
training_model = model
# make prediction model
prediction_model = retinanet_bbox(model=model)
# compile model
training_model.compile(loss={
'regression':
losses.iou_loss(args.loss, args.loss_weight),
'classification':
losses.focal(),
'centerness':
losses.bce(),
},
optimizer=keras.optimizers.adam(lr=lr))
return model, training_model, prediction_model
def main():
from pprint import pprint
config = B2Config()
# pprint(vars(config))
# exit()
batch_size = 4
num_classes = 15
epochs = 5
steps_per_epoch = 1000
parser = Parser(
config=config,
batch_size=batch_size,
num_classes=num_classes)
training_model = efficientdet(config, num_classes, weights=None)
# compile model
training_model.compile(
optimizer=Adam(lr=1e-3),
loss={
'regression': smooth_l1(),
'classification': focal()})
# print(training_model.summary())
# # create the callbacks
# callbacks = create_callbacks(
# model,
# prediction_model,
# validation_generator,
# args,
# )
train_dataset_function = parser.get_dataset(filenames='./DATA/train*.tfrecord')
training_model.fit(
train_dataset_function,
epochs=epochs,
steps_per_epoch=steps_per_epoch,)
# callbacks=callbacks)
os.makedirs("./checkpoints", exist_ok=True)
training_model.save("./checkpoints/efficientdetB2_final")
def __init__(self, backbone):
# a dictionary mapping custom layer names to the correct classes
self.custom_objects = {
'UpsampleLike': layers.UpsampleLike,
'PriorProbability': initializers.PriorProbability,
'RegressBoxes': layers.RegressBoxes,
'FilterDetections': layers.FilterDetections,
'Anchors': layers.Anchors,
'ClipBoxes': layers.ClipBoxes,
'_focal': losses.focal(),
'bce_': losses.bce(),
'iou_': losses.iou(),
}
self.backbone = backbone
self.validate()
def main(args=None):
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
# make sure keras is the minimum required version
check_keras_version()
# optionally choose specific GPU
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
keras.backend.tensorflow_backend.set_session(get_session())
# make save path if it doesn't exist
if args.save_path is not None and not os.path.exists(args.save_path):
os.makedirs(args.save_path)
# create the generator
generator = create_generator(args)
# load the model
print('Loading model, this may take a second...')
model = keras.models.load_model(args.model, custom_objects=custom_objects)
# print model summary
print(model.summary())
loss = {'regression': losses.smooth_l1(), 'classification': losses.focal()}
# start evaluation
average_precisions = evaluate(
generator,
model,
# loss,
iou_threshold=args.iou_threshold,
score_threshold=args.score_threshold,
max_detections=args.max_detections,
save_path=args.save_path)
# print evaluation
for label, average_precision in average_precisions.items():
print(generator.label_to_name(label),
'{:.4f}'.format(average_precision))
print('mAP: {:.4f}'.format(
sum(average_precisions.values()) / len(average_precisions)))
def _create_models(self,
backbone_retinanet,
num_classes,
weights,
freeze_backbone=False,
lr=1e-5):
"""
Creates three models (model, training_model, prediction_model).
Parameters
----------
backbone_retinanet : A function to call to create a retinanet model with a given backbone.
num_classes : The number of classes to train.
weights : The weights to load into the model.
multi_gpu : The number of GPUs to use for training.
freeze_backbone : If True, disables learning for the backbone.
config : Config parameters, None indicates the default configuration.
Returns
-------
model : The base model.
training_model : The training model. If multi_gpu=0, this is identical to model.
prediction_model : The model wrapped with utility functions to perform object detection (applies regression values and performs NMS).
"""
modifier = freeze_model if freeze_backbone else None
anchor_params = None
num_anchors = None
model = self._model_with_weights(backbone_retinanet(
num_classes, num_anchors=num_anchors, modifier=modifier),
weights=weights,
skip_mismatch=True)
training_model = model
prediction_model = retinanet_bbox(model=model,
anchor_params=anchor_params)
training_model.compile(loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=lr,
clipnorm=0.001))
return model, training_model, prediction_model
def __init__(self, backbone):
import custom_layers as layers
import losses
import initializers
self.custom_objects = {
'PriorProbability': initializers.PriorProbability,
'UpsampleLike': layers.UpsampleLike,
'RegressBoxes': layers.RegressBoxes,
'FilterDetections': layers.FilterDetections,
'Anchors': layers.Anchors,
'ClipBoxes': layers.ClipBoxes,
# 'BatchNormalization': layers.BatchNormalization,
'_smooth_l1': losses.smooth_l1(),
'_focal': losses.focal(),
}
self.backbone = backbone
self.validate()
def train(self, epochs, backbone_name, evaluation):
#Compile model
self.model.compile(
loss={
'regression': losses.iou(),
'classification': losses.focal(),
'centerness': losses.bce(),
},
optimizer=keras.optimizers.adam(lr=1e-5)
# optimizer=keras.optimizers.sgd(lr=1e-5, momentum=0.9, decay=1e-5, nesterov=True)
)
# create the generators
train_generator, validation_generator = create_generators(
self.config, self.dataset)
# create the callbacks
callbacks = create_callbacks(
self.config,
backbone_name,
self.model,
self.training_model,
self.prediction_model,
validation_generator,
evaluation,
self.log_dir,
)
# start training
return self.training_model.fit_generator(
generator=train_generator,
initial_epoch=0,
steps_per_epoch=self.config.STEPS_PER_EPOCH,
epochs=epochs,
verbose=1,
callbacks=callbacks,
max_queue_size=10,
validation_data=validation_generator)
def level_select(cls_pred, regr_pred, gt_boxes, feature_shapes, strides, pos_scale=0.2):
"""
Args:
cls_pred: (sum(fh * fw), num_classes)
regr_pred: (sum(fh * fw), 4)
gt_boxes: (MAX_NUM_GT_BOXES, 5)
feature_shapes: (5, 2)
strides:
pos_scale:
Returns:
"""
gt_labels = tf.cast(gt_boxes[:, 4], tf.int32)
gt_boxes = gt_boxes[:, :4]
focal_loss = focal()
iou_loss = iou()
gt_boxes, non_zeros = trim_zeros_graph(gt_boxes)
num_gt_boxes = tf.shape(gt_boxes)[0]
gt_labels = tf.boolean_mask(gt_labels, non_zeros)
level_losses = []
for level_id in range(len(strides)):
stride = strides[level_id]
fh = feature_shapes[level_id][0]
fw = feature_shapes[level_id][1]
fa = tf.reduce_prod(feature_shapes, axis=-1)
start_idx = tf.reduce_sum(fa[:level_id])
end_idx = start_idx + fh * fw
cls_pred_i = tf.reshape(cls_pred[start_idx:end_idx], (fh, fw, tf.shape(cls_pred)[-1]))
regr_pred_i = tf.reshape(regr_pred[start_idx:end_idx], (fh, fw, tf.shape(regr_pred)[-1]))
proj_boxes = gt_boxes / stride
x1, y1, x2, y2 = prop_box_graph(proj_boxes, pos_scale, fw, fh)
def compute_gt_box_loss(args):
x1_ = args[0]
y1_ = args[1]
x2_ = args[2]
y2_ = args[3]
gt_box = args[4]
gt_label = args[5]
locs_cls_pred_i = cls_pred_i[y1_:y2_, x1_:x2_, :]
locs_cls_pred_i = tf.reshape(locs_cls_pred_i, (-1, tf.shape(locs_cls_pred_i)[-1]))
locs_cls_true_i = tf.zeros_like(locs_cls_pred_i)
gt_label_col = tf.ones_like(locs_cls_true_i[:, 0:1])
locs_cls_true_i = tf.concat([locs_cls_true_i[:, :gt_label],
gt_label_col,
locs_cls_true_i[:, gt_label + 1:],
], axis=-1)
loss_cls = focal_loss(K.expand_dims(locs_cls_true_i, axis=0), K.expand_dims(locs_cls_pred_i, axis=0))
locs_regr_pred_i = regr_pred_i[y1_:y2_, x1_:x2_, :]
locs_regr_pred_i = tf.reshape(locs_regr_pred_i, (-1, tf.shape(locs_regr_pred_i)[-1]))
locs_x = K.arange(x1_, x2_, dtype=tf.float32)
locs_y = K.arange(y1_, y2_, dtype=tf.float32)
shift_x = (locs_x + 0.5) * stride
shift_y = (locs_y + 0.5) * stride
shift_xx, shift_yy = tf.meshgrid(shift_x, shift_y)
shift_xx = tf.reshape(shift_xx, (-1,))
shift_yy = tf.reshape(shift_yy, (-1,))
shifts = K.stack((shift_xx, shift_yy, shift_xx, shift_yy), axis=-1)
l = tf.maximum(shifts[:, 0] - gt_box[0], 0)
t = tf.maximum(shifts[:, 1] - gt_box[1], 0)
r = tf.maximum(gt_box[2] - shifts[:, 2], 0)
b = tf.maximum(gt_box[3] - shifts[:, 3], 0)
locs_regr_true_i = tf.stack([l, t, r, b], axis=-1)
locs_regr_true_i /= 4.0
loss_regr = iou_loss(K.expand_dims(locs_regr_true_i, axis=0), K.expand_dims(locs_regr_pred_i, axis=0))
return loss_cls + loss_regr
level_loss = tf.map_fn(
compute_gt_box_loss,
elems=[x1, y1, x2, y2, gt_boxes, gt_labels],
dtype=tf.float32
)
level_losses.append(level_loss)
losses = tf.stack(level_losses, axis=-1)
gt_box_levels = tf.argmin(losses, axis=-1)
padding_gt_box_levels = tf.ones((MAX_NUM_GT_BOXES - num_gt_boxes), dtype=tf.int64) * -1
gt_box_levels = tf.concat([gt_box_levels, padding_gt_box_levels], axis=0)
return gt_box_levels
def main(args=None):
"""
Train an EfficientPose model.
Args:
args: parseargs object containing configuration for the training procedure.
"""
allow_gpu_growth_memory()
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
# create the generators
print("\nCreating the Generators...")
train_generator, validation_generator = create_generators(args)
print("Done!")
num_rotation_parameters = train_generator.get_num_rotation_parameters()
num_classes = train_generator.num_classes()
num_anchors = train_generator.num_anchors
# optionally choose specific GPU
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
print("\nBuilding the Model...")
model, prediction_model, all_layers = build_EfficientPose(
args.phi,
num_classes=num_classes,
num_anchors=num_anchors,
freeze_bn=not args.no_freeze_bn,
score_threshold=args.score_threshold,
num_rotation_parameters=num_rotation_parameters)
print("Done!")
# load pretrained weights
if args.weights:
if args.weights == 'imagenet':
model_name = 'efficientnet-b{}'.format(args.phi)
file_name = '{}_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5'.format(
model_name)
file_hash = WEIGHTS_HASHES[model_name][1]
weights_path = keras.utils.get_file(file_name,
BASE_WEIGHTS_PATH + file_name,
cache_subdir='models',
file_hash=file_hash)
model.load_weights(weights_path, by_name=True)
else:
print('Loading model, this may take a second...')
custom_load_weights(filepath=args.weights,
layers=all_layers,
skip_mismatch=True)
print("\nDone!")
# freeze backbone layers
if args.freeze_backbone:
# 227, 329, 329, 374, 464, 566, 656
for i in range(1, [227, 329, 329, 374, 464, 566, 656][args.phi]):
model.layers[i].trainable = False
# compile model
model.compile(
optimizer=Adam(lr=args.lr, clipnorm=0.001),
loss={
'regression':
smooth_l1(),
'classification':
focal(),
'transformation':
transformation_loss(model_3d_points_np=train_generator.
get_all_3d_model_points_array_for_loss(),
num_rotation_parameter=num_rotation_parameters)
},
loss_weights={
'regression': 1.0,
'classification': 1.0,
'transformation': 0.02
})
# create the callbacks
callbacks = create_callbacks(
model,
prediction_model,
validation_generator,
args,
)
if not args.compute_val_loss:
validation_generator = None
elif args.compute_val_loss and validation_generator is None:
raise ValueError(
'When you have no validation data, you should not specify --compute-val-loss.'
)
# start training
return model.fit_generator(generator=train_generator,
steps_per_epoch=args.steps,
initial_epoch=0,
epochs=args.epochs,
verbose=1,
callbacks=callbacks,
workers=args.workers,
use_multiprocessing=args.multiprocessing,
max_queue_size=args.max_queue_size,
validation_data=validation_generator)
def build_meta_select_target(cls_pred,
regr_pred,
gt_boxes,
feature_shapes,
strides,
shrink_ratio=0.2):
gt_labels = tf.cast(gt_boxes[:, 4], tf.int32)
gt_boxes = gt_boxes[:, :4]
max_gt_boxes = tf.shape(gt_boxes)[0]
focal_loss = focal()
iou_loss = iou()
gt_boxes, non_zeros = trim_padding_boxes(gt_boxes)
num_gt_boxes = tf.shape(gt_boxes)[0]
gt_labels = tf.boolean_mask(gt_labels, non_zeros)
level_losses = []
for level_id in range(len(strides)):
stride = strides[level_id]
fh = feature_shapes[level_id][0]
fw = feature_shapes[level_id][1]
fa = tf.reduce_prod(feature_shapes, axis=-1)
start_idx = tf.reduce_sum(fa[:level_id])
end_idx = start_idx + fh * fw
cls_pred_i = tf.reshape(cls_pred[start_idx:end_idx],
(fh, fw, tf.shape(cls_pred)[-1]))
regr_pred_i = tf.reshape(regr_pred[start_idx:end_idx],
(fh, fw, tf.shape(regr_pred)[-1]))
# (num_gt_boxes, )
x1, y1, x2, y2 = shrink_and_project_boxes(gt_boxes,
fw,
fh,
stride,
shrink_ratio=shrink_ratio)
def compute_gt_box_loss(args):
x1_ = args[0]
y1_ = args[1]
x2_ = args[2]
y2_ = args[3]
gt_box = args[4]
gt_label = args[5]
def do_match_pixels_in_level():
locs_cls_pred_i = cls_pred_i[y1_:y2_, x1_:x2_, :]
locs_cls_pred_i = tf.reshape(
locs_cls_pred_i, (-1, tf.shape(locs_cls_pred_i)[-1]))
locs_cls_true_i = tf.zeros_like(locs_cls_pred_i)
gt_label_col = tf.ones_like(locs_cls_true_i[:, 0:1])
locs_cls_true_i = tf.concat([
locs_cls_true_i[:, :gt_label],
gt_label_col,
locs_cls_true_i[:, gt_label + 1:],
],
axis=-1)
loss_cls = focal_loss(tf.expand_dims(locs_cls_true_i, axis=0),
tf.expand_dims(locs_cls_pred_i, axis=0))
locs_regr_pred_i = regr_pred_i[y1_:y2_, x1_:x2_, :]
locs_regr_pred_i = tf.reshape(
locs_regr_pred_i, (-1, tf.shape(locs_regr_pred_i)[-1]))
locs_x = tf.cast(tf.range(x1_, x2_), dtype=tf.float32)
locs_y = tf.cast(tf.range(y1_, y2_), dtype=tf.float32)
shift_x = (locs_x + 0.5) * stride
shift_y = (locs_y + 0.5) * stride
shift_xx, shift_yy = tf.meshgrid(shift_x, shift_y)
shift_xx = tf.reshape(shift_xx, (-1, ))
shift_yy = tf.reshape(shift_yy, (-1, ))
shifts = tf.stack((shift_xx, shift_yy, shift_xx, shift_yy),
axis=-1)
l = tf.maximum(shifts[:, 0] - gt_box[0], 0)
t = tf.maximum(shifts[:, 1] - gt_box[1], 0)
r = tf.maximum(gt_box[2] - shifts[:, 2], 0)
b = tf.maximum(gt_box[3] - shifts[:, 3], 0)
locs_regr_true_i = tf.stack([l, t, r, b], axis=-1)
locs_regr_true_i = locs_regr_true_i / 4.0 / stride
loss_regr = iou_loss(tf.expand_dims(locs_regr_true_i, axis=0),
tf.expand_dims(locs_regr_pred_i, axis=0))
return loss_cls + loss_regr
def do_not_match_pixels_in_level():
box_loss = tf.constant(1e7, dtype=tf.float32)
return box_loss
level_box_loss = tf.cond(
tf.equal(tf.cast(x1_, tf.int32), tf.cast(x2_, tf.int32))
| tf.equal(tf.cast(y1_, tf.int32), tf.cast(y2_, tf.int32)),
do_not_match_pixels_in_level, do_match_pixels_in_level)
return level_box_loss
level_loss = tf.map_fn(compute_gt_box_loss,
elems=[x1, y1, x2, y2, gt_boxes, gt_labels],
dtype=tf.float32)
level_losses.append(level_loss)
losses = tf.stack(level_losses, axis=-1)
gt_box_levels = tf.argmin(losses, axis=-1, output_type=tf.int32)
padding_gt_box_levels = tf.ones(
(max_gt_boxes - num_gt_boxes), dtype=tf.int32) * -1
gt_box_levels = tf.concat([gt_box_levels, padding_gt_box_levels], axis=0)
return gt_box_levels
def main(args=None):
# create object that stores backbone information
backbone = models.backbone(args.backbone)
# create the generators
train_generator, validation_generator = create_generators(
args, backbone.preprocess_image)
# create the model
if args.snapshot is not None:
print('Loading model, this may take a second...')
model = models.load_model(args.snapshot, backbone_name=args.backbone)
training_model = model
anchor_params = None
if args.config and 'anchor_parameters' in args.config:
anchor_params = parse_anchor_parameters(args.config)
prediction_model = retinanet_bbox(model=model,
anchor_params=anchor_params)
# compile model
training_model.compile(
loss={
'regression': losses.iou_loss(args.loss, args.loss_weight),
'classification': losses.focal(),
'centerness': losses.bce(),
},
optimizer=keras.optimizers.Adam(lr=1e-5)
# optimizer=keras.optimizers.sgd(lr=1e-5, momentum=0.9, decay=1e-5, nesterov=True)
)
else:
weights = args.weights
# default to imagenet if nothing else is specified
if weights is None and args.imagenet_weights:
weights = backbone.download_imagenet()
print('Creating model, this may take a second...')
model, training_model, prediction_model = create_models(
backbone_retinanet=backbone.retinanet,
num_classes=train_generator.num_classes(),
weights=weights,
num_gpus=args.num_gpus,
freeze_backbone=args.freeze_backbone,
lr=args.lr,
config=args.config,
args=args)
parallel_model = multi_gpu_model(training_model, gpus=2)
parallel_model.compile(loss={
'regression':
losses.iou_loss(args.loss, args.loss_weight),
'classification':
losses.focal(),
'centerness':
losses.bce(),
},
optimizer=keras.optimizers.Adam(lr=1e-4))
# print model summary
# print(model.summary())
# this lets the generator compute backbone layer shapes using the actual backbone model
if 'vgg' in args.backbone or 'densenet' in args.backbone:
train_generator.compute_shapes = make_shapes_callback(model)
if validation_generator:
validation_generator.compute_shapes = train_generator.compute_shapes
# create the callbacks
callbacks = create_callbacks(
model,
training_model,
prediction_model,
validation_generator,
args,
)
if not args.compute_val_loss:
validation_generator = None
# start training
parallel_model.fit_generator(generator=train_generator,
steps_per_epoch=len(train_generator),
epochs=args.epochs,
verbose=1,
callbacks=callbacks,
validation_data=validation_generator)
def main(args=None):
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
# create object that stores backbone information
backbone = models.backbone(args.backbone)
# make sure keras is the minimum required version
check_keras_version()
# optionally choose specific GPU
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
keras.backend.tensorflow_backend.set_session(get_session())
# optionally load config parameters
if args.config:
args.config = read_config_file(args.config)
# create the generators
train_generator, validation_generator = create_generators(
args, backbone.preprocess_image)
# create the model
if args.snapshot is not None:
print('Loading model, this may take a second...')
model = models.load_model(args.snapshot, backbone_name=args.backbone)
training_model = model
anchor_params = None
if args.config and 'anchor_parameters' in args.config:
anchor_params = parse_anchor_parameters(args.config)
prediction_model = retinanet_bbox(model=model,
anchor_params=anchor_params)
# compile model
training_model.compile(
loss={
'regression': losses.iou(),
'classification': losses.focal(),
'centerness': losses.bce(),
},
optimizer=keras.optimizers.adam(lr=1e-5)
# optimizer=keras.optimizers.sgd(lr=1e-5, momentum=0.9, decay=1e-5, nesterov=True)
)
else:
weights = args.weights
# default to imagenet if nothing else is specified
if weights is None and args.imagenet_weights:
weights = backbone.download_imagenet()
print('Creating model, this may take a second...')
model, training_model, prediction_model = create_models(
backbone_retinanet=backbone.retinanet,
num_classes=train_generator.num_classes(),
weights=weights,
num_gpus=args.num_gpus,
freeze_backbone=args.freeze_backbone,
lr=args.lr,
config=args.config)
# print model summary
# print(model.summary())
# this lets the generator compute backbone layer shapes using the actual backbone model
if 'vgg' in args.backbone or 'densenet' in args.backbone:
train_generator.compute_shapes = make_shapes_callback(model)
if validation_generator:
validation_generator.compute_shapes = train_generator.compute_shapes
# create the callbacks
callbacks = create_callbacks(
model,
training_model,
prediction_model,
validation_generator,
args,
)
if not args.compute_val_loss:
validation_generator = None
# start training
return training_model.fit_generator(
generator=train_generator,
initial_epoch=0,
steps_per_epoch=args.steps,
epochs=args.epochs,
verbose=1,
callbacks=callbacks,
workers=args.workers,
use_multiprocessing=args.multiprocessing,
max_queue_size=args.max_queue_size,
validation_data=validation_generator)
def create_models(fl_gamma,
fl_alpha,
r_weight,
c_weight,
p_weight,
train_type,
sample_t,
backbone_retinanet,
num_classes,
weights,
class_weights,
loss_weights,
multi_gpu=0,
freeze_backbone=False,
lr=1e-5,
config=None):
""" Creates three models (model, training_model, prediction_model).
Args
backbone_retinanet : A function to call to create a retinanet model with a given backbone.
num_classes : The number of classes to train.
weights : The weights to load into the model.
multi_gpu : The number of GPUs to use for training.
freeze_backbone : If True, disables learning for the backbone.
config : Config parameters, None indicates the default configuration.
Returns
model : The base model. This is also the model that is saved in snapshots.
training_model : The training model. If multi_gpu=0, this is identical to model.
prediction_model : The model wrapped with utility functions to perform object detection (applies regression values and performs NMS).
"""
modifier = freeze_model if freeze_backbone else None
# load anchor parameters, or pass None (so that defaults will be used)
anchor_params = None
num_anchors = None
if config and 'anchor_parameters' in config:
anchor_params = parse_anchor_parameters(config)
num_anchors = anchor_params.num_anchors()
# Keras recommends initialising a multi-gpu model on the CPU to ease weight sharing, and to prevent OOM errors.
# optionally wrap in a parallel model
if multi_gpu > 1:
from keras.utils import multi_gpu_model
with tf.device('/cpu:0'):
model = model_with_weights(backbone_retinanet(
num_classes, num_anchors=num_anchors, modifier=modifier),
weights=weights,
skip_mismatch=True)
training_model = multi_gpu_model(model, gpus=multi_gpu)
else:
model = model_with_weights(backbone_retinanet(num_classes,
train_type,
num_anchors=num_anchors,
modifier=modifier),
weights=weights,
skip_mismatch=True)
training_model = model
# make prediction model
prediction_model = retinanet_bbox(model=model, anchor_params=anchor_params)
if train_type == "single":
# compile model
training_model.compile(
loss={
'regression':
losses.smooth_l1(r_weight),
## HERE THE INPUT ARGUMENTS CAN BE GIVEN: default focal(alpha=0.25, gamma=2.0)
## gamma: the actual "focusing parameter"
'classification':
losses.focal(c_weight, fl_alpha, fl_gamma, class_weights)
},
optimizer=keras.optimizers.adam(lr=lr, clipnorm=0.001),
#sample_weight_mode="temporal",
#sample_weights=sample_t
)
elif train_type == "multi":
training_model.compile(
loss={
'regression':
losses.smooth_l1(r_weight),
## HERE THE INPUT ARGUMENTS CAN BE GIVEN: default focal(alpha=0.25, gamma=2.0)
## gamma: the actual "focusing parameter"
'classification_artefact':
losses.focal(c_weight, fl_alpha, fl_gamma),
'classification_polyp':
losses.focal2(p_weight, fl_alpha, fl_gamma)
},
optimizer=keras.optimizers.adam(lr=lr, clipnorm=0.001)
#loss_weights=loss_weights
)
return model, training_model, prediction_model
def efficientdet_training(config, LOCAL_ANNOTATIONS_PATH, LOCAL_ROOT_PATH,
LOCAL_CLASSES_PATH, LOCAL_VALIDATIONS_PATH,
LOCAL_LOGS_PATH, LOCAL_SNAPSHOTS_PATH):
(model, prediction_model, train_generator, evaluation_generator,
validation_generator, config) = load_efficient_det(
config, LOCAL_ANNOTATIONS_PATH, LOCAL_ROOT_PATH, LOCAL_CLASSES_PATH,
LOCAL_VALIDATIONS_PATH, LOCAL_LOGS_PATH, LOCAL_SNAPSHOTS_PATH)
#model.summary()
# create the callbacks
if config['validation']:
callbacks = create_callbacks(model, prediction_model,
evaluation_generator,
validation_generator, LOCAL_LOGS_PATH,
LOCAL_SNAPSHOTS_PATH, config)
else:
callbacks = create_callbacks(model, prediction_model,
evaluation_generator, train_generator,
LOCAL_LOGS_PATH, LOCAL_SNAPSHOTS_PATH,
config)
model.compile(optimizer=Adam(lr=1e-3),
loss={
'regression': smooth_l1(),
'classification': focal(gamma=config['focal_gamma'])
})
os.makedirs(LOCAL_LOGS_PATH, exist_ok=True)
with open(os.path.join(LOCAL_LOGS_PATH, 'config.yaml'), 'w') as file:
yaml.dump(config, file)
with open(LOCAL_LOGS_PATH + '/stdout.txt', 'w') as f:
with redirect_stdout(f):
print('Path : ', LOCAL_ROOT_PATH)
start = datetime.now()
print(f'Started training at: {start}')
if not validation_generator:
results = model.fit_generator(
generator=train_generator,
steps_per_epoch=config['steps_per_epoch'],
initial_epoch=0,
epochs=config['nb_epoch'],
verbose=1,
callbacks=callbacks,
workers=config['workers'],
use_multiprocessing=config['multiprocessing'],
max_queue_size=config['max_queue_size'])
else:
results = model.fit_generator(
generator=train_generator,
steps_per_epoch=config['steps_per_epoch'],
initial_epoch=0,
epochs=config['nb_epoch'],
verbose=1,
callbacks=callbacks,
workers=config['workers'],
use_multiprocessing=config['multiprocessing'],
max_queue_size=config['max_queue_size'],
validation_data=validation_generator)
end = datetime.now()
print(f'Completed training at: {end}')
print(f'Total training time: {diff(start, end)}')
def convert(args):
rot_parameters = {"axis_angle": 3, "rotation_matrix": 9, "quaternion": 4}
model = model_clean.EfficientPose(
args.phi,
num_classes=args.num_classes,
num_anchors=args.num_anchors,
freeze_bn=False,
score_threshold=args.score_threshold,
num_rotation_parameters=rot_parameters[args.rotation_representation],
lite=args.lite,
no_se=args.no_se,
use_groupnorm=args.use_groupnorm)
_, _, tflite_model = model.get_models()
tflite_model(
[tf.random.normal((1, 512, 512, 3)),
tf.random.normal((1, 6))])
# inp = tf.keras.layers.Input((512,512,3), dtype=tf.uint8)
# inp2 = tf.keras.layers.Input((6,))
# tflite_raw_model = tfkeras.EfficientNetB0(input_tensor=preprocess_image(inp))
# tflite_raw_model = tf.keras.Model(inputs=[inp, inp2], outputs=tflite_raw_model)
weight_loader.load_weights_rec(tflite_model, args.weights)
if args.freeze_bn:
weight_loader.freeze_bn(tflite_model)
if args.q_aware:
gen, val = create_generators(args)
q_aware_model = tfmot.quantization.keras.quantize_model(tflite_model)
q_aware_model.compile(
optimizer="adam",
loss={
'regression':
smooth_l1(),
'classification':
focal(),
'transformation':
transformation_loss(model_3d_points_np=gen.
get_all_3d_model_points_array_for_loss(),
num_rotation_parameter=3)
},
loss_weights={
'regression': 1.0,
'classification': 1.0,
'transformation': 0.02
})
q_aware_model.fit(gen,
steps_per_epoch=1000,
initial_epoch=0,
epochs=5,
verbose=1,
validation_data=val)
#tflite_raw_model.save("models/model.h5")
tflite_model(
[tf.random.normal((1, 512, 512, 3)),
tf.random.normal((1, 6))])
converter = tf.lite.TFLiteConverter.from_keras_model(tflite_model)
converter.experimental_new_converter = True
converter.target_spec.supported_ops = [
tf.lite.OpsSet.TFLITE_BUILTINS, # enable TensorFlow Lite ops.
tf.lite.OpsSet.SELECT_TF_OPS # enable TensorFlow ops.
]
#converter.representative_dataset = tf.lite.RepresentativeDataset(create_generators(args))
#converter.optimizations = [tf.lite.Optimize.DEFAULT]
converter.allow_custom_ops = True
tflite_model = converter.convert()
outfile = os.path.join(
args.models_dir,
f"{args.base_name}_phi{args.phi}{'_lite' if args.lite else ''}.tflite")
os.makedirs(args.models_dir, exist_ok=True)
with open(outfile, 'wb') as f:
f.write(tflite_model)
if args.benchmark:
run_benchmark.benchmark(args)
def main(config_file=None):
cwd = os.getcwd()
# parse configuration file
if config_file is None:
config_file = sys.argv[-1]
config_file = os.path.join(cwd, config_file)
config_file_name = config_file.split('/')[-1]
configs = parse_config(config_file)
# save config file
if configs['Train']['save_configs']:
# confirm save dir
config_save_path = 'logs/' + configs['Name']
config_save_path = os.path.join(cwd, config_save_path)
if not os.path.exists(config_save_path):
os.mkdir(os.path.join(cwd, config_save_path))
# copy config file
config_dst_name = configs['Name'] + '.json'
config_file_dst = os.path.join(config_save_path, config_dst_name)
shutil.copy(config_file, config_file_dst)
# make sure keras is the minimum required version
check_keras_version()
# optionally choose specific GPU
if configs['Train']['gpu']:
os.environ['CUDA_VISIBLE_DEVICES'] = configs['Train']['gpu']
keras.backend.tensorflow_backend.set_session(get_session())
# create the generators
train_generator, validation_generator = create_generators(configs)
# create the model
if configs['Train']['load_snapshot'] is not None:
print('Loading model, this may take a second...')
model = models.load_model(configs['Train']['load_snapshot'],
backbone=configs['Train']['backbone'])
training_model = prediction_model = model
else:
weights = configs['Train']['weights']
# default to imagenet if nothing else is specified
if weights is None and configs['Train']['imagenet_weights']:
weights = models.download_imagenet(configs['Train']['backbone'])
print('Creating model, this may take a second...')
backbone = configs['Train']['backbone']
num_classes = train_generator.num_classes()
multi_gpu = configs['Train']['multi_gpu']
freeze_backbone = configs['Train']['freeze_backbone']
modifier = freeze_model if freeze_backbone else None
# Keras recommends initialising a multi-gpu model on the CPU to ease weight sharing, and to prevent OOM errors.
# optionally wrap in a parallel model
if multi_gpu > 1:
with tf.device('/cpu:0'):
retinanet = models.retinanet_backbone(
configs['Train']['backbone'])(num_classes,
backbone=backbone,
modifier=modifier)
model = model_with_weights(retinanet,
weights=weights,
skip_mismatch=True)
training_model = multi_gpu_model(model, gpus=multi_gpu)
else:
retinanet = models.retinanet_backbone(
configs['Train']['backbone'])(num_classes,
backbone=backbone,
modifier=modifier)
training_model = model = model_with_weights(retinanet,
weights=weights,
skip_mismatch=True)
# make prediction model
prediction_model = retinanet_bbox(model=model,
anchor_param=configs['Anchors'])
# compile model
subnet_loss = {
'regression': losses.smooth_l1(),
'classification': losses.focal()
}
# run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
# run_metadata = tf.RunMetadata()
if configs['Train']['lr_multiplier_layer']:
optimizer = AdamWithLRMult(
lr=configs['Train']['init_lr'],
lr_multipliers=configs['Train']['lr_multiplier_layer'],
debug_verbose=False,
clipnorm=0.001)
else:
optimizer = keras.optimizers.adam(configs['Train']['init_lr'],
clipnorm=0.001)
training_model.compile(loss=subnet_loss, optimizer=optimizer)
# training_model.compile(loss=subnet_loss, optimizer=keras.optimizers.adam(configs['Train']['init_lr'], clipnorm=0.001))
# this lets the generator compute backbone layer shapes using the actual backbone model
if 'vgg' in configs['Train']['backbone'] or 'densenet' in configs['Train'][
'backbone']:
compute_anchor_targets = functools.partial(
anchor_targets_bbox, shapes_callback=make_shapes_callback(model))
train_generator.compute_anchor_targets = compute_anchor_targets
if validation_generator is not None:
validation_generator.compute_anchor_targets = compute_anchor_targets
# create the callbacks
callbacks = create_callbacks(
model,
training_model,
prediction_model,
validation_generator,
configs,
)
# start training
training_model.fit_generator(
train_generator,
validation_data=validation_generator,
validation_steps=39,
steps_per_epoch=configs['Train']['steps'],
epochs=configs['Train']['epochs'],
verbose=1,
callbacks=callbacks,
)
def __init__(self, t=1.0, alpha=1.0, reduce=True):
super(KDLoss, self).__init__()
self.t = t
self.alpha = alpha
self.focal = losses.focal(reduce=True)
self.reduce = reduce
import losses
import utils.customized_optimizer
import numpy as np
"""
A dictionary mapping custom layer names to the correct classes.
"""
custom_objects = {
'UpsampleLike': layers.UpsampleLike,
'PriorProbability': initializers.PriorProbability,
'RegressBoxes': layers.RegressBoxes,
'FilterDetections': layers.FilterDetections,
'Anchors': layers.Anchors,
'ClipBoxes': layers.ClipBoxes,
'_smooth_l1': losses.smooth_l1(),
'_focal': losses.focal(),
'AdamWithLRMult': utils.customized_optimizer.AdamWithLRMult
}
def default_classification_model(num_classes,
num_anchors,
pyramid_feature_size=256,
prior_probability=0.01,
classification_feature_size=256,
name='classification_submodel'):
""" Creates the default regression submodel.
Args
num_classes : Number of classes to predict a score for at each feature level.
num_anchors : Number of anchors to predict classification scores for at each feature level.
def main():
backbone = models.backbone('resnet50')
# create the generators
#train_generator, validation_generator = create_generators(args, backbone.preprocess_image)
random_transform = True
val_annotations = './data/processed/val.csv'
annotations = './data/processed/train.csv'
classes = './data/processed/classes.csv'
common_args = {
'batch_size': 8,
'image_min_side': 224,
'image_max_side': 1333,
'preprocess_image': backbone.preprocess_image,
}
# create random transform generator for augmenting training data
if random_transform:
transform_generator = random_transform_generator(
min_rotation=-0.05,
max_rotation=0.05,
min_translation=(-0.1, -0.1),
max_translation=(0.1, 0.1),
#min_shear=-0.1,
#max_shear=0.1,
min_scaling=(0.8, 0.8),
max_scaling=(1.2, 1.2),
flip_x_chance=0.5,
#flip_y_chance=0.5,
)
else:
transform_generator = random_transform_generator(flip_x_chance=0.5)
train_generator = CSVGenerator(annotations,
classes,
transform_generator=transform_generator,
**common_args)
if val_annotations:
validation_generator = CSVGenerator(val_annotations, classes,
**common_args)
else:
validation_generator = None
#train_generator, validation_generator = create_generators(args, backbone.preprocess_image)
num_classes = 1 # change
model = backbone.retinanet(num_classes, backbone='resnet50')
training_model = model
# prediction_model = retinanet_bbox(model=model)
nms = True
class_specific_filter = True
name = 'retinanet-bbox'
anchor_params = AnchorParameters.default
# compute the anchors
features = [
model.get_layer(p_name).output
for p_name in ['P3', 'P4', 'P5', 'P6', 'P7']
]
anchor = [
layers.Anchors(size=anchor_params.sizes[i],
stride=anchor_params.strides[i],
ratios=anchor_params.ratios,
scales=anchor_params.scales,
name='anchors_{}'.format(i))(f)
for i, f in enumerate(features)
]
anchors = keras.layers.Concatenate(axis=1, name='anchors')(anchor)
# we expect the anchors, regression and classification values as first output
regression = model.outputs[0] # check
classification = model.outputs[1]
# "other" can be any additional output from custom submodels, by default this will be []
other = model.outputs[2:]
# apply predicted regression to anchors
boxes = layers.RegressBoxes(name='boxes')([anchors, regression])
boxes = layers.ClipBoxes(name='clipped_boxes')([model.inputs[0], boxes])
# filter detections (apply NMS / score threshold / select top-k)
detections = layers.FilterDetections(
nms=nms,
class_specific_filter=class_specific_filter,
name='filtered_detections')([boxes, classification] + other)
outputs = detections
# construct the model
prediction_model = keras.models.Model(inputs=model.inputs,
outputs=outputs,
name=name)
# end of prediction_model = retinanet_bbox(model=model)
# compile model
training_model.compile(loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.SGD(lr=1e-2,
momentum=0.9,
decay=.0001,
nesterov=True,
clipnorm=1)
# , clipnorm=0.001)
)
print(model.summary())
# start of create_callbacks
#callbacks = create_callbacks(model,training_model,prediction_model,validation_generator,args,)
callbacks = []
tensorboard_callback = None
tensorboard_callback = keras.callbacks.TensorBoard(
log_dir='',
histogram_freq=0,
batch_size=8,
write_graph=True,
write_grads=False,
write_images=False,
embeddings_freq=0,
embeddings_layer_names=None,
embeddings_metadata=None)
callbacks.append(tensorboard_callback)
evaluation = Evaluate(validation_generator,
tensorboard=tensorboard_callback,
weighted_average=False)
evaluation = RedirectModel(evaluation, prediction_model)
callbacks.append(evaluation)
makedirs('./snapshots/')
checkpoint = keras.callbacks.ModelCheckpoint(os.path.join(
'./snapshots/', '{backbone}_{dataset_type}_{{epoch:02d}}.h5'.format(
backbone='resnet50', dataset_type='csv')),
verbose=1,
save_best_only=False,
monitor="mAP",
mode='max')
checkpoint = RedirectModel(checkpoint, model)
callbacks.append(checkpoint)
callbacks.append(
keras.callbacks.ReduceLROnPlateau(monitor='loss',
factor=0.9,
patience=4,
verbose=1,
mode='auto',
min_delta=0.0001,
cooldown=0,
min_lr=0))
steps = 2500
epochs = 25
# start training
history = training_model.fit(
generator=train_generator,
steps_per_epoch=steps,
epochs=epochs,
verbose=1,
callbacks=callbacks,
)
timestr = time.strftime("%Y-%m-%d-%H%M")
history_path = os.path.join(
'./snapshots/', '{timestr}_{backbone}.csv'.format(timestr=timestr,
backbone='resnet50',
dataset_type='csv'))
pd.DataFrame(history.history).to_csv(history_path)
def main(args=None):
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
# create the generators
train_generator, validation_generator = create_generators(args)
num_classes = train_generator.num_classes()
num_anchors = train_generator.num_anchors
# optionally choose specific GPU
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
K.set_session(get_session())
model, prediction_model = efficientdet(
args.phi,
num_classes=num_classes,
num_anchors=num_anchors,
weighted_bifpn=args.weighted_bifpn,
freeze_bn=args.freeze_bn,
detect_quadrangle=args.detect_quadrangle)
# load pretrained weights
if args.snapshot:
if args.snapshot == 'imagenet':
model_name = 'efficientnet-b{}'.format(args.phi)
file_name = '{}_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5'.format(
model_name)
file_hash = WEIGHTS_HASHES[model_name][1]
weights_path = keras.utils.get_file(file_name,
BASE_WEIGHTS_PATH + file_name,
cache_subdir='models',
file_hash=file_hash)
model.load_weights(weights_path, by_name=True)
else:
print('Loading model, this may take a second...')
model.load_weights(args.snapshot, by_name=True)
# freeze backbone layers
if args.freeze_backbone:
# 227, 329, 329, 374, 464, 566, 656
for i in range(1, [227, 329, 329, 374, 464, 566, 656][args.phi]):
model.layers[i].trainable = False
if args.gpu and len(args.gpu.split(',')) > 1:
model = keras.utils.multi_gpu_model(model,
gpus=list(
map(int, args.gpu.split(','))))
# compile model
model.compile(
optimizer=Adam(lr=1e-3),
loss={
'regression':
smooth_l1_quad() if args.detect_quadrangle else smooth_l1(),
'classification':
focal()
},
)
# print(model.summary())
# create the callbacks
callbacks = create_callbacks(
model,
prediction_model,
validation_generator,
args,
)
if not args.compute_val_loss:
validation_generator = None
elif args.compute_val_loss and validation_generator is None:
raise ValueError(
'When you have no validation data, you should not specify --compute-val-loss.'
)
# start training
return model.fit_generator(generator=train_generator,
steps_per_epoch=args.steps,
initial_epoch=0,
epochs=args.epochs,
verbose=1,
callbacks=callbacks,
workers=args.workers,
use_multiprocessing=args.multiprocessing,
max_queue_size=args.max_queue_size,
validation_data=validation_generator)
def main(train_csv, val_csv, classes_csv, epoch_num, phi_num, steps_epoch,
batch_num, model_path):
batch_size = batch_num #def 1
phi = phi_num #def 0
is_text_detect = False
is_detect_quadrangle = False
rand_transf_augm = True
train_ann_path = train_csv
train_class_path = classes_csv
val_ann_path = val_csv
val_class_path = classes_csv
epochs = epoch_num
workers = 1
steps_p_epoch = steps_epoch
use_multiproc = True
max_que_size = 10
comp_loss = True
gpu = 0
freeze_bn_arg = True
weighted_bi = False
# create the generators
train_generator, validation_generator = create_generators(
batch_size, phi, is_text_detect, is_detect_quadrangle,
rand_transf_augm, train_ann_path, val_ann_path, train_class_path,
val_class_path)
num_classes = train_generator.num_classes()
num_anchors = train_generator.num_anchors
# optionally choose specific GPU
if gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = gpu
# K.set_session(get_session())
model, prediction_model = efficientdet(
phi,
num_classes=num_classes,
num_anchors=num_anchors,
weighted_bifpn=weighted_bi,
freeze_bn=freeze_bn_arg,
detect_quadrangle=is_detect_quadrangle)
# freeze backbone layers
if freeze_bn_arg:
# 227, 329, 329, 374, 464, 566, 656
for i in range(1, [227, 329, 329, 374, 464, 566, 656][phi]):
model.layers[i].trainable = False
model.compile(
optimizer=Adam(lr=1e-3),
loss={
'regression':
smooth_l1_quad() if is_detect_quadrangle else smooth_l1(),
'classification': focal()
},
)
# print(model.summary())
# create the callbacks
callbacks = create_callbacks(model_path)
if not comp_loss:
validation_generator = None
elif comp_loss and validation_generator is None:
raise ValueError(
'When you have no validation data, you should not specify --compute-val-loss.'
)
# start training
return model.fit_generator(generator=train_generator,
steps_per_epoch=steps_p_epoch,
initial_epoch=0,
epochs=epochs,
verbose=1,
callbacks=callbacks,
workers=workers,
use_multiprocessing=use_multiproc,
max_queue_size=max_que_size,
validation_data=validation_generator)
