def create_models(backbone_retinanet,
num_classes,
weights,
multi_gpu=0,
freeze_backbone=False):
modifier = freeze_model if freeze_backbone else None
# Keras recommends initialising a multi-gpu model on the CPU to ease weight sharing, and to prevent OOM errors.
# optionally wrap in a parallel model
if multi_gpu > 1:
with tf.device('/cpu:0'):
model = model_with_weights(backbone_retinanet(num_classes,
modifier=modifier),
weights=weights,
skip_mismatch=True)
training_model = multi_gpu_model(model, gpus=multi_gpu)
else:
model = model_with_weights(backbone_retinanet(num_classes,
modifier=modifier),
weights=weights,
skip_mismatch=True)
training_model = model
# make prediction model
prediction_model = retinanet_bbox(model=model, class_specific_filter=False)
# compile model
training_model.compile(loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=1e-5,
clipnorm=0.001))
return model, training_model, prediction_model
def create_models(num_classes, p):
# create "base" model (no NMS)
image = keras.layers.Input((None, None, 3))
if p['resnet'] == 101:
model = ResNet101RetinaNet(image, num_classes=num_classes, weights=p['weights'], nms=False)
elif p['resnet'] == 152:
model = ResNet152RetinaNet(image, num_classes=num_classes, weights=p['weights'], nms=False)
else: # 50
model = ResNet50RetinaNet(image, num_classes=num_classes, weights=p['weights'], nms=False)
training_model = model
# append NMS for prediction only
classification = model.outputs[1]
detections = model.outputs[2]
boxes = keras.layers.Lambda(lambda x: x[:, :, :4])(detections)
detections = layers.NonMaximumSuppression(name='nms')([boxes, classification, detections])
prediction_model = keras.models.Model(inputs=model.inputs, outputs=model.outputs[:2] + [detections])
# compile model
training_model.compile(
loss={
'regression' : losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr = p['learning-rate'], clipnorm=0.001),
metrics = []
)
return model, training_model, prediction_model
def test_backbone(backbone):
# ignore warnings in this test
warnings.simplefilter('ignore')
num_classes = 10
inputs = np.zeros((1, 200, 400, 3), dtype=np.float32)
targets = [
np.zeros((1, 14814, 5), dtype=np.float32),
np.zeros((1, 14814, num_classes))
]
inp = keras.layers.Input(inputs[0].shape)
densenet_backbone = DenseNetBackbone(backbone)
model = densenet_backbone.retinanet(num_classes=num_classes, inputs=inp)
model.summary()
# compile model
model.compile(loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=1e-5, clipnorm=0.001))
model.fit(inputs, targets, batch_size=1)
def create_models(num_classes, weights='imagenet', multi_gpu=0):
# create "base" model (no NMS)
image = keras.layers.Input((None, None, 4))
# Keras recommends initialising a multi-gpu model on the CPU to ease weight sharing, and to prevent OOM errors.
# optionally wrap in a parallel model
if multi_gpu > 1:
with tf.device('/cpu:0'):
model = ResNet50RetinaNet(image, num_classes=num_classes, weights=weights, nms=False)
training_model = multi_gpu_model(model, gpus=multi_gpu)
else:
model = ResNet50RetinaNet(image, num_classes=num_classes, weights=weights, nms=False)
training_model = model
# append NMS for prediction only
classification = model.outputs[1]
detections = model.outputs[2]
boxes = keras.layers.Lambda(lambda x: x[:, :, :4])(detections)
detections = layers.NonMaximumSuppression(name='nms')([boxes, classification, detections])
prediction_model = keras.models.Model(inputs=model.inputs, outputs=model.outputs[:2] + [detections])
# compile model
training_model.compile(
loss={
'regression' : losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=1e-5, clipnorm=0.001)
)
return model, training_model, prediction_model
def test_backbone(backbone, alpha):
# ignore warnings in this test
warnings.simplefilter('ignore')
num_classes = 10
inputs = np.zeros((1, 1024, 363, 3), dtype=np.float32)
targets = [
np.zeros((1, 68760, 5), dtype=np.float32),
np.zeros((1, 68760, num_classes + 1))
]
inp = keras.layers.Input(inputs[0].shape)
mobilenet_backbone = MobileNetBackbone(
backbone='{}_{}'.format(backbone, format(alpha)))
training_model = mobilenet_backbone.retinanet(num_classes=num_classes,
inputs=inp)
training_model.summary()
# compile model
training_model.compile(loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.Adam(lr=1e-5,
clipnorm=0.001))
training_model.fit(inputs, targets, batch_size=1)
def load_model(weights, nms_args={}, compile=False):
from keras.models import model_from_yaml
from keras.optimizers import Adam
from keras_retinanet.losses import smooth_l1, focal
from keras_retinanet.models.resnet import custom_objects
model_dir = os.path.dirname(weights)
config_dir = model_dir
config = os.path.join(config_dir, 'config.yml')
while not os.path.exists(config):
config_dir = os.path.dirname(config_dir)
config = os.path.join(config_dir, 'config.yml')
if os.path.relpath(config, 'models/config.yml') == '.':
raise FileNotFoundError('Model config not found.')
print('Load model config %s' % config)
with open(config, 'r') as f:
config = f.read()
custom_objects['non_max_suppression'] = non_max_suppression(nms_args)
model = model_from_yaml(config, custom_objects=custom_objects)
if compile:
loss = dict(regression=smooth_l1(), classification=focal())
optimizer = Adam(lr=1e-5, clipnorm=0.001)
model.compile(loss=loss, optimizer=optimizer)
model.load_weights(weights)
return model
def create_models(backbone_retinanet,
num_classes,
weights,
multi_gpu=0,
freeze_backbone=False):
""" Creates three models (model, training_model, prediction_model).
Args
backbone_retinanet : A function to call to create a retinanet model with a given backbone.
num_classes : The number of classes to train.
weights : The weights to load into the model.
multi_gpu : The number of GPUs to use for training.
freeze_backbone : If True, disables learning for the backbone.
Returns
model : The base model. This is also the model that is saved in snapshots.
training_model : The training model. If multi_gpu=0, this is identical to model.
prediction_model : The model wrapped with utility functions to perform object detection (applies regression values and performs NMS).
"""
modifier = freeze_model if freeze_backbone else None
# Keras recommends initialising a multi-gpu model on the CPU to ease weight sharing, and to prevent OOM errors.
# optionally wrap in a parallel model
if multi_gpu > 1:
from keras.utils import multi_gpu_model
with tf.device('/cpu:0'):
model = model_with_weights(backbone_retinanet(num_classes,
modifier=modifier),
weights=weights,
skip_mismatch=True)
training_model = multi_gpu_model(model, gpus=multi_gpu)
else:
model = model_with_weights(backbone_retinanet(num_classes,
modifier=modifier),
weights=weights,
skip_mismatch=True)
training_model = model
# make prediction model
prediction_model = retinanet_bbox(model=model)
# compile model
training_model.compile(
loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.sgd(lr=1e-2,
momentum=0.9,
decay=.0001,
nesterov=True,
clipnorm=1) #, clipnorm=0.001)
)
return model, training_model, prediction_model
def create_models(backbone_retinanet, num_classes, weights, multi_gpu=0,
freeze_backbone=False, lr=1e-5, config=None, regularisation=3.0):
""" Creates three models (model, training_model, prediction_model).
Args
backbone_retinanet : A function to call to create a retinanet model with a given backbone.
num_classes : The number of classes to train.
weights : The weights to load into the model.
multi_gpu : The number of GPUs to use for training.
freeze_backbone : If True, disables learning for the backbone.
config : Config parameters, None indicates the default configuration.
Returns
model : The base model. This is also the model that is saved in snapshots.
training_model : The training model. If multi_gpu=0, this is identical to model.
prediction_model : The model wrapped with utility functions to perform object detection (applies regression values and performs NMS).
"""
modifier = freeze_model if freeze_backbone else None
# load anchor parameters, or pass None (so that defaults will be used)
anchor_params = None
num_anchors = None
if config and 'anchor_parameters' in config:
anchor_params = parse_anchor_parameters(config)
num_anchors = anchor_params.num_anchors()
# Keras recommends initialising a multi-gpu model on the CPU to ease weight sharing, and to prevent OOM errors.
# optionally wrap in a parallel model
if multi_gpu > 1:
from keras.utils import multi_gpu_model
with tf.device('/cpu:0'):
model = model_with_weights(backbone_retinanet(num_classes, num_anchors=num_anchors, modifier=modifier), weights=weights, skip_mismatch=True)
training_model = multi_gpu_model(model, gpus=multi_gpu)
else:
model = model_with_weights(backbone_retinanet(num_classes, num_anchors=num_anchors, modifier=modifier), weights=weights, skip_mismatch=True)
training_model = model
# make prediction model
prediction_model = retinanet_bbox(model=model, anchor_params=anchor_params)
# compile model
if regularisation != 3.0:
print(f"regularisation = {regularisation}")
training_model.compile(
loss={
'regression' : losses.smooth_l1(regularisation),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=lr, clipnorm=0.001)
)
return model, training_model, prediction_model
def create_model():
model_path = Path(
'/data/students_home/fschipani/thesis/keras-retinanet/snapshots/resnet50_coco_best_v2.1.0.h5'
)
model = models.backbone('resnet50').retinanet(
num_classes=train_generator.num_classes())
model.load_weights(model_path, by_name=True, skip_mismatch=True)
model.compile(loss={
'regression': smooth_l1(),
'classification': focal()
},
optimizer=keras.optimizers.adam(lr=1e-5, clipnorm=0.001))
return model
def create_models(backbone_retinanet,
backbone,
num_classes,
weights,
multi_gpu=0,
freeze_backbone=False):
modifier = freeze_model if freeze_backbone else None
# Keras recommends initialising a multi-gpu model on the CPU to ease weight sharing, and to prevent OOM errors.
# optionally wrap in a parallel model
if multi_gpu > 1:
with tf.device('/cpu:0'):
model = model_with_weights(backbone_retinanet(num_classes,
backbone=backbone,
nms=False,
modifier=modifier),
weights=weights,
skip_mismatch=True)
training_model = multi_gpu_model(model, gpus=multi_gpu)
# append NMS for prediction only
classification = model.outputs[1]
detections = model.outputs[2]
boxes = keras.layers.Lambda(lambda x: x[:, :, :4])(detections)
detections = layers.NonMaximumSuppression(name='nms')(
[boxes, classification, detections])
prediction_model = keras.models.Model(inputs=model.inputs,
outputs=model.outputs[:2] +
[detections])
else:
model = model_with_weights(backbone_retinanet(num_classes,
backbone=backbone,
nms=True,
modifier=modifier),
weights=weights,
skip_mismatch=True)
training_model = model
prediction_model = model
# compile model
training_model.compile(loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=1e-5,
clipnorm=0.001))
return model, training_model, prediction_model
def __create_models(self, backbone_retinanet, num_classes):
anchor_params = self._anchor_params
num_anchors = anchor_params.num_anchors()
model = backbone_retinanet(num_classes,
num_anchors=num_anchors,
modifier=None)
training_model = model
prediction_model = retinanet_bbox(model=model,
anchor_params=anchor_params)
training_model.compile(loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=1e-5,
clipnorm=0.001))
return model, training_model, prediction_model
def main(args=None):
args = parse_args(args)
model = keras.models.load_model(args.model_in,
custom_objects={
'UpsampleLike': UpsampleLike,
'PriorProbability': PriorProbability,
'RegressBoxes': RegressBoxes,
'FilterDetections': FilterDetections,
'Anchors': Anchors,
'ClipBoxes': ClipBoxes,
'_smooth_l1': smooth_l1(),
'_focal': focal(),
})
assert(all(output in model.output_names for output in ['regression', 'classification'])), \
"Input is not a training model (no 'regression' and 'classification' outputs were found, outputs are: {}).".format(model.output_names)
model = convert_model(model)
model.save(args.model_out)
def vision_model_losses(output, config):
if 'retinanet' in output:
if 'retinanet_regression' in output:
try:
weight = {'retinanet_regression': float(output.split(':')[1])}
except:
weight = {'retinanet_regression': 1.0}
return {
'retinanet_regression': retinanet_losses.smooth_l1()
}, weight
if 'retinanet_classification' in output:
try:
weight = {
'retinanet_classification': float(output.split(':')[1])
}
except:
weight = {'retinanet_classification': 1.0}
return {
'retinanet_classification': retinanet_losses.focal()
}, weight
return {}, {}
def create_models(num_classes, weights='imagenet', multi_gpu=0):
# create "base" model (no NMS)
image = keras.layers.Input((None, None, 3))
# Keras recommends initialising a multi-gpu model on the CPU to ease weight sharing, and to prevent OOM errors.
# optionally wrap in a parallel model
if multi_gpu > 1:
with tf.device('/cpu:0'):
model = ResNet50RetinaNet(image,
num_classes=num_classes,
weights=weights,
nms=False)
training_model = multi_gpu_model(model, gpus=multi_gpu)
else:
model = ResNet50RetinaNet(image,
num_classes=num_classes,
weights=weights,
nms=False)
training_model = model
# append NMS for prediction only
classification = model.outputs[1]
detections = model.outputs[2]
boxes = keras.layers.Lambda(lambda x: x[:, :, :4])(detections)
detections = layers.NonMaximumSuppression(name='nms')(
[boxes, classification, detections])
prediction_model = keras.models.Model(inputs=model.inputs,
outputs=model.outputs[:2] +
[detections])
# compile model
training_model.compile(loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=1e-5,
clipnorm=0.001))
return model, training_model, prediction_model
def build_model(backbone, nms_args, weights=None):
from keras.layers import Input, Lambda
from keras.optimizers import Adam
from keras.models import Model
from keras_retinanet.models.resnet import resnet_retinanet
from keras_retinanet.losses import smooth_l1, focal
from keras_retinanet.layers import RegressBoxes
inp = Input([None, None, 3])
model, anchors = resnet_retinanet(2, backbone, bbox=False, inputs=inp)
regression, classification = model.outputs
boxes = RegressBoxes(name='boxes')([anchors, regression])
detections = Lambda(non_max_suppression(),
name='detections',
arguments=nms_args)([boxes, classification])
model = Model(inp, [regression, classification, detections],
name='RetinaNet')
loss = dict(regression=smooth_l1(), classification=focal())
optimizer = Adam(lr=1e-5, clipnorm=0.001)
model.compile(loss=loss, optimizer=optimizer)
if weights: model.load_weights(weights)
return model
def test_backbone(backbone, alpha):
# ignore warnings in this test
warnings.simplefilter('ignore')
num_classes = 10
inputs = np.zeros((1, 1024, 363, 3), dtype=np.float32)
targets = [np.zeros((1, 70776, 5), dtype=np.float32), np.zeros((1, 70776, num_classes))]
inp = keras.layers.Input(inputs[0].shape)
training_model = mobilenet_retinanet(num_classes=num_classes, backbone='{}_{}'.format(backbone, format(alpha)), inputs=inp)
training_model.summary()
# compile model
training_model.compile(
loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=1e-5, clipnorm=0.001))
training_model.fit(inputs, targets, batch_size=1)
def test_backbone(backbone):
# ignore warnings in this test
warnings.simplefilter('ignore')
num_classes = 10
inputs = np.zeros((1, 200, 400, 3), dtype=np.float32)
targets = [np.zeros((1, 14814, 5), dtype=np.float32), np.zeros((1, 14814, num_classes + 1))]
inp = keras.layers.Input(inputs[0].shape)
densenet_backbone = DenseNetBackbone(backbone)
model = densenet_backbone.retinanet(num_classes=num_classes, inputs=inp)
model.summary()
# compile model
model.compile(
loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=1e-5, clipnorm=0.001))
model.fit(inputs, targets, batch_size=1)
print('Creating model, this may take a second...')
modifier = freeze_model if freeze_backbone else None
model = backbone_model.retinanet(train_generator.num_classes(),
modifier=modifier)
if weights is not None:
model.load_weights(weights, by_name=True, skip_mismatch=True)
training_model = model
# make prediction model
prediction_model = retinanet_bbox(model=model)
# compile model
training_model.compile(loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=1e-5,
clipnorm=0.001))
# print model summary
print(model.summary())
# this lets the generator compute backbone layer shapes using the actual backbone model
if 'vgg' in backbone or 'densenet' in backbone:
train_generator.compute_shapes = make_shapes_callback(model)
if validation_generator:
validation_generator.compute_shapes = train_generator.compute_shapes
callbacks = []
def get_losses(args):
return {'regression': losses.smooth_l1(), 'classification': losses.focal()}
def train_model(concepts,
model_name,
collectionIds,
min_examples,
epochs,
download_data=True):
classmap = get_classmap(concepts)
# Downloads the annotation data and saves it into training and validation csv's.
# Also downloads corresponding images.
if download_data:
folders = ["weights"]
for dir in folders:
if os.path.exists(dir):
shutil.rmtree(dir)
os.makedirs(dir)
start = time.time()
print("Starting Download.")
download_annotations(min_examples, collectionIds, concepts, classmap,
img_folder, train_annot_file, valid_annot_file)
end = time.time()
print("Done Downloading Annotations: " + str((end - start) / 60) +
" minutes")
# Trains the model!!!!! WOOOT WOOOT!
start = time.time()
print("Starting Training.")
# Suggested to initialize model on cpu before turning into a
# multi_gpu model to save gpu memory
with tf.device('/cpu:0'):
model = models.backbone('resnet50').retinanet(
num_classes=len(concepts)) #modifier=freeze_model)
model.load_weights(weights_path, by_name=True, skip_mismatch=True)
gpus = len(
[i for i in device_lib.list_local_devices() if i.device_type == 'GPU'])
if gpus > 1:
training_model = multi_gpu_model(model, gpus=gpus)
else:
training_model = model
training_model.compile(loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=1e-5,
clipnorm=0.001))
# transform_generator = random_transform_generator(
# min_rotation=-0.1,
# max_rotation=0.1,
# min_translation=(-0.1, -0.1),
# max_translation=(0.1, 0.1),
# min_shear=-0.1,
# max_shear=0.1,
# min_scaling=(0.9, 0.9),
# max_scaling=(1.1, 1.1),
# flip_x_chance=0.5,
# flip_y_chance=0.5,
# )
temp = pd.DataFrame(list(zip(classmap.values(), classmap.keys())))
temp.to_csv('classmap.csv', index=False, header=False)
train_generator = custom(
train_annot_file,
'classmap.csv',
# transform_generator=transform_generator,
batch_size=batch_size)
test_generator = CSVGenerator(valid_annot_file,
'classmap.csv',
batch_size=batch_size,
shuffle_groups=False)
# Checkpoint: save models that are improvements
checkpoint = ModelCheckpoint(weights_path,
monitor='val_loss',
save_best_only=True)
checkpoint = RedirectModel(checkpoint, model)
#stopping: stops training if val_loss stops improving
stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=10)
log_table_name = 'previous_runs'
# Initialize a log entry in the previous_runs table
# TODO: put this in progress_callbacks.py, it makes more sense there
# tb_log_id = create_log_entry(
# table_name=log_table_name,
# model_name=model_name,
# users=users,
# videos=videos,
# min_examples=min_examples,
# concepts=selected_concepts,
# epochs=epochs
# )
# Every epoch upload tensorboard logs to the S3 bucket
# log_callback = TensorBoardLog(id_=tb_log_id, table_name=log_table_name)
# tensorboard_callback = keras.callbacks.TensorBoard(
# log_dir=f'./logs/{tb_log_id}', histogram_freq=0, batch_size=batch_size,
# write_graph=True, write_grads=False, write_images=False,
# embeddings_freq=0, embeddings_layer_names=None,
# embeddings_metadata=None, embeddings_data=None, update_freq='epoch')
# Every batch and epoch update a database table with the current progress
progress_callback = Progress(steps_per_epoch=len(train_generator),
num_epochs=epochs)
history = training_model.fit_generator(
train_generator,
epochs=epochs,
callbacks=[checkpoint, stopping, progress_callback], #, log_callback],
validation_data=test_generator,
verbose=2).history
s3.upload_file(weights_path, S3_BUCKET,
S3_BUCKET_WEIGHTS_FOLDER + model_name + ".h5")
end = time.time()
print("Done Training Model: " + str((end - start) / 60) + " minutes")
from tensorflow import keras
from keras_retinanet.visualization import draw_box, draw_caption, label_color
from keras_retinanet.util import UpsampleLike, RegressBoxes, FilterDetections, PriorProbability, ClipBoxes
from keras_retinanet.losses import smooth_l1, focal
from keras_retinanet.anchors import Anchors
from keras_retinanet.image import read_image_bgr, preprocess_image, resize_image
import matplotlib.pyplot as plt
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
model = keras.models.load_model('out.h5', custom_objects={
'UpsampleLike' : UpsampleLike,
'PriorProbability' : PriorProbability,
'RegressBoxes' : RegressBoxes,
'FilterDetections' : FilterDetections,
'Anchors' : Anchors,
'ClipBoxes' : ClipBoxes,
'_smooth_l1' : smooth_l1(),
'_focal' : focal(),
}, compile=False)
label_to_name = [
'naruto_uzumaki',
'sasuke_uchiha',
'kakashi_hatake',
'gaara',
'itachi_uchiha',
'deidara',
'minato_namikaze',
'shikamaru_nara',
'hinata_hyuuga',
'sakura_haruno',
'sai',
def main():
parser = ConfigParser(interpolation=ExtendedInterpolation())
parser.read("config.ini")
params = parser["train"]
backbone = params["backbone"]
# use this environment flag to change which GPU to use
# os.environ["CUDA_VISIBLE_DEVICES"] = "1"
# set the modified tf session as backend in keras
keras.backend.tensorflow_backend.set_session(get_session())
resume = bool(params["resume"])
# if resuming load saved model instead
if resume:
model = models.load_model(params["model_path"], backbone_name=backbone)
else:
model = models.backbone(backbone).retinanet(
num_classes=int(params["num_classes"]))
model.compile(loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=1e-5, clipnorm=0.001),
metrics=['accuracy'])
weights = params["weights_path"]
model.load_weights(weights, by_name=True, skip_mismatch=True)
batch_size = int(params["batchsize"])
preprocesss = get_preprocessing(bool(params["preprocess"]))
anno_path = "retinanet_annotations/{}".format(params["dataset"])
data = CSVGenerator(os.path.join(anno_path, "train_annotation.csv"),
os.path.join(anno_path, "class_map.csv"),
batch_size=batch_size,
image_min_side=512,
image_max_side=512,
preprocess_image=preprocesss)
val_data = CSVGenerator(os.path.join(anno_path, "val_annotation.csv"),
os.path.join(anno_path, "class_map.csv"),
batch_size=batch_size,
image_min_side=512,
image_max_side=512,
preprocess_image=preprocesss)
val_dataset_size = val_data.size()
# TODO: Set the file path under which you want to save the model.
current_time = datetime.now().strftime('%Y-%m-%d %H:%M').split(" ")
model_checkpoint = ModelCheckpoint(filepath=os.path.join(
params["check_point_path"],
'retinanet_{}_{}_{}_{}_{}.h5'.format(params["dataset"], backbone,
params["preprocess"],
current_time[0],
current_time[1])),
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='auto',
period=int(params["save_period"]))
# model_checkpoint.best =
csv_logger = CSVLogger(filename=os.path.join(
params["csv_path"],
'retinanet_{}_{}_{}_{}_{}.csv'.format(params["dataset"], backbone,
params["preprocess"],
current_time[0],
current_time[1])),
separator=',',
append=True)
# learning_rate_scheduler = LearningRateScheduler(schedule=lr_schedule, verbose=1)
terminate_on_nan = TerminateOnNaN()
tensorboard = TensorBoard(log_dir=os.path.join(params["tensorboard_path"],
'retinanet',
current_time[0],
current_time[1]),
write_images=True,
write_graph=True)
callbacks = [
model_checkpoint,
csv_logger,
# learning_rate_scheduler,
terminate_on_nan,
tensorboard,
]
# Fit model
# If you're resuming a previous training, set `initial_epoch` and `final_epoch` accordingly.
if resume:
initial_epoch = int(params["initial_epoch"])
final_epoch = int(params["final_epoch"])
else:
initial_epoch = 0
final_epoch = int(params["epochs"])
steps_per_epoch = int(params["steps_per_epoch"])
model.fit_generator(
generator=data,
steps_per_epoch=steps_per_epoch,
epochs=final_epoch,
callbacks=callbacks,
validation_data=val_data,
validation_steps=ceil(val_dataset_size / batch_size),
initial_epoch=initial_epoch,
)
def create_models(backbone_retinanet,
num_classes,
weights,
multi_gpu=0,
freeze_backbone=False,
lr=1e-5,
config=None,
targets=None,
freeze_layers=0,
modifier=None):
"""Creates three models (model, training_model, prediction_model).
Args:
backbone_retinanet : A function to call to create a retinanet model
with a given backbone.
num_classes : The number of classes to train.
weights : The weights to load into the model.
multi_gpu : The number of GPUs to use for training.
freeze_backbone : If True, disables learning for the backbone.
config : Config parameters, None indicates the default configuration.
targets : Target tensors if training a model with tfrecord inputs
freeze_layers : int layer number to freeze from bottom of the retinanet
network during finetuning. e.g. 10 will set
layers 0:10 to layer.trainable = False. 0 is default, no freezing.
modifier : function that takes in a model and freezes resnet layers,
returns modified object
Returns:
model : The base model.
This is also the model that is saved in snapshots.
training_model : The training model.
If multi_gpu=0, this is identical to model.
prediction_model : The model wrapped with utility functions to perform
object detection (applies regression values and performs NMS).
"""
# if not modifier:
modifier = freeze_model if freeze_backbone else None
# load anchor parameters, or pass None (so that defaults will be used)
anchor_params = None
num_anchors = None
if config and 'anchor_parameters' in config:
anchor_params = parse_anchor_parameters(config)
num_anchors = anchor_params.num_anchors()
# Keras recommends initialising a multi-gpu model on the CPU to ease weight sharing,
# and to prevent OOM errors.
# optionally wrap in a parallel model
if multi_gpu > 1:
from keras.utils import multi_gpu_model
with tf.device('/cpu:0'):
model = model_with_weights(backbone_retinanet(
num_classes, num_anchors=num_anchors, modifier=modifier),
weights=weights,
skip_mismatch=True)
training_model = multi_gpu_model(model, gpus=multi_gpu)
else:
model = model_with_weights(backbone_retinanet(num_classes,
num_anchors=num_anchors,
modifier=modifier),
weights=weights,
skip_mismatch=True)
training_model = model
# make prediction model
prediction_model = retinanet_bbox(model=model, anchor_params=anchor_params)
# Compile model
if targets:
# tfdataset target tensor from tfrecords pipelione
training_model.compile(loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=lr,
clipnorm=0.001),
target_tensors=targets)
else:
# compile model
training_model.compile(loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=lr,
clipnorm=0.001))
return model, training_model, prediction_model
def train_model(concepts,
verify_videos,
model_name,
collection_ids,
min_examples,
epochs,
download_data=True,
verified_only=False,
include_tracking=True):
""" Trains the model, uploads its weights, and determines best
confidence thresholds for predicting
"""
# Generate a random unique ID for this training job
job_id = uuid.uuid4().hex
model, training_model = _initilize_model(len(concepts))
num_workers = _get_num_workers()
_redirect_outputs(job_id)
training_model.compile(
loss={
'regression': losses.smooth_l1(),
'classification': losses.focal()
},
optimizer=keras.optimizers.adam(lr=1e-5, clipnorm=0.001)
)
print("initializing annotation generator")
annotation_generator = AnnotationGenerator(
collection_ids=collection_ids,
verified_only=verified_only,
include_tracking=include_tracking,
min_examples=min_examples,
classes=concepts,
verify_videos=verify_videos
)
# Get model name & version to insert into model_versions table
# a user dictionary containing the concept counts from this model
# for each userid
model_name_str = model_name.split("-")[0]
model_version = model_name.split("-")[-1]
insert_counts_query = '''
UPDATE
model_versions
SET
concept_count = %s
WHERE
model = %s
AND
version = %s
'''
query(
insert_counts_query,
(json.dumps(annotation_generator.userDict), model_name_str, model_version)
)
train_generator = annotation_generator.flow_from_s3(
image_folder=config.IMAGE_FOLDER,
subset='training',
batch_size=config.BATCH_SIZE
)
test_generator = annotation_generator.flow_from_s3(
image_folder=config.IMAGE_FOLDER,
subset='validation',
batch_size=config.BATCH_SIZE
)
callbacks = _get_callbacks(
model=model,
job_id=job_id,
model_name=model_name,
collection_ids=collection_ids,
min_examples=min_examples,
epochs=epochs,
steps_per_epoch=len(train_generator)
)
training_model.fit_generator(
train_generator,
epochs=epochs,
callbacks=callbacks,
validation_data=test_generator,
use_multiprocessing=True,
workers=num_workers,
verbose=2
)
model.save(config.WEIGHTS_PATH)
# Upload the weights file to the S3 bucket
_upload_weights(model_name)
# Evaluate the best confidence thresholds for the model
evaluate_class_thresholds(model, test_generator)
