def test_model_backup_loading(shapes_image_size, shapes_sample_config,
shapes_temp_dir):
"""Test the model checkpoint recovering
"""
config = read_config(shapes_sample_config)
label_ids = [x['id'] for x in config['labels'] if x['is_evaluate']]
cnn = FeatureDetectionNetwork("test",
image_size=shapes_image_size,
nb_labels=len(label_ids))
model = Model(cnn.X, cnn.Y)
old_weights = model.get_weights()
checkpoint_path = os.path.join(str(shapes_temp_dir), "checkpoints")
if os.path.isdir(checkpoint_path):
checkpoints = os.listdir(checkpoint_path)
if len(checkpoints) > 0:
model_checkpoint = max(checkpoints)
trained_model_epoch = int(model_checkpoint[-5:-3])
checkpoint_complete_path = os.path.join(checkpoint_path,
model_checkpoint)
model.load_weights(checkpoint_complete_path)
else:
trained_model_epoch = 0
new_weights = model.get_weights()
assert trained_model_epoch > 0
assert len(old_weights) == len(new_weights)
assert old_weights[0].shape == new_weights[0].shape
# Test if old and new weights are different (at least for one layer)
assert any(not np.allclose(lhs, rhs)
for lhs, rhs in zip(old_weights, new_weights))
def init_model(
problem, instance_name, image_size, nb_labels, dropout, network
):
"""Initialize a convolutional neural network with Keras API starting from a
set of parameters
Parameters
----------
problem : str
Type of solved problem, either `feature_detection` or
`semantic_segmentation`
instance_name : str
Name of the instance, for identification purpose
image_size : int
Image size, in pixels (height=width)
nb_labels : int
Number of output labels
dropout : float
Dropout rate
network : str
Network architecture
Returns
-------
keras.models.Model
Convolutional neural network
"""
K.clear_session()
if problem == "feature_detection":
net = FeatureDetectionNetwork(
network_name=instance_name,
image_size=image_size,
nb_labels=nb_labels,
dropout=dropout,
architecture=network,
)
elif problem == "semantic_segmentation":
net = SemanticSegmentationNetwork(
network_name=instance_name,
image_size=image_size,
nb_labels=nb_labels,
dropout=dropout,
architecture=network,
)
else:
logger.error(
(
"Unrecognized model. Please enter 'feature_detection' "
"or 'semantic_segmentation'."
)
)
sys.exit(1)
return Model(net.X, net.Y)
def test_simple_network_architecture(shapes_image_size, nb_channels, shapes_nb_labels):
"""Test a simple feature detection network
"""
net = FeatureDetectionNetwork("fd_test", image_size=shapes_image_size,
nb_channels=nb_channels, nb_labels=shapes_nb_labels)
m = Model(net.X, net.Y)
input_shape = m.input_shape
output_shape = m.output_shape
assert len(input_shape) == 4
assert input_shape[1:] == (shapes_image_size, shapes_image_size, nb_channels)
assert len(output_shape) == 2
assert output_shape[1] == shapes_nb_labels
def test_model_training(
shapes_image_size,
shapes_sample,
shapes_sample_config,
shapes_temp_dir,
shapes_nb_images,
):
"""Test the training of a simple neural network with Keras API, as well as
model inference and trained model backup
One big test function to avoid duplicating the training operations (that
can be long)
"""
BATCH_SIZE = 10
NB_EPOCHS = 1
NB_STEPS = shapes_nb_images // BATCH_SIZE
config = read_config(shapes_sample_config)
label_ids = [x["id"] for x in config["labels"] if x["is_evaluate"]]
gen = create_generator(
"shapes",
"featdet",
shapes_sample,
shapes_image_size,
BATCH_SIZE,
config["labels"],
)
cnn = FeatureDetectionNetwork("test",
image_size=shapes_image_size,
nb_labels=len(label_ids))
model = Model(cnn.X, cnn.Y)
model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["acc"])
hist = model.fit_generator(gen, epochs=NB_EPOCHS, steps_per_epoch=NB_STEPS)
assert len(hist.history) == 2
assert all(k in hist.history.keys() for k in ["acc", "loss"])
assert hist.history["acc"][0] >= 0 and hist.history["acc"][0] <= 1
test_image = np.random.randint(
0, 255, [BATCH_SIZE, shapes_image_size, shapes_image_size, 3])
score = model.predict(test_image)
assert score.shape == (BATCH_SIZE, len(label_ids))
assert all(0 <= s and s <= 1 for s in score.ravel())
BACKUP_FILENAME = os.path.join(
str(shapes_temp_dir),
"checkpoints",
"test_model_{:02d}.h5".format(NB_EPOCHS),
)
model.save(BACKUP_FILENAME)
assert os.path.isfile(BACKUP_FILENAME)
def test_inception_network_architecture(mapillary_image_size, nb_channels, mapillary_nb_labels):
"""Test a Inception-inspired feature detection network
"""
net = FeatureDetectionNetwork("fd_test", image_size=mapillary_image_size,
nb_channels=nb_channels, nb_labels=mapillary_nb_labels,
architecture="inception")
m = Model(net.X, net.Y)
input_shape = m.input_shape
output_shape = m.output_shape
assert len(input_shape) == 4
assert input_shape[1:] == (mapillary_image_size, mapillary_image_size, nb_channels)
assert len(output_shape) == 2
assert output_shape[1] == mapillary_nb_labels
def run_model(
train_generator,
validation_generator,
dl_model,
output_folder,
instance_name,
image_size,
nb_labels,
nb_epochs,
nb_training_image,
nb_validation_image,
batch_size,
dropout,
network,
learning_rate,
learning_rate_decay,
):
"""Run deep learning `dl_model` starting from training and validation data
generators, depending on a range of hyperparameters
Parameters
----------
train_generator : generator
Training data generator
validation_generator : generator
Validation data generator
dl_model : str
Name of the addressed research problem (*e.g.* `feature_detection` or
`semantic_segmentation`)
output_folder : str
Name of the folder where the trained model will be stored on the file
system
instance_name : str
Name of the instance
image_size : int
Size of images, in pixel (height=width)
nb_labels : int
Number of labels into the dataset
nb_epochs : int
Number of epochs during which models will be trained
nb_training_image : int
Number of images into the training dataset
nb_validation_image : int
Number of images into the validation dataset
batch_size : int
Number of images into each batch
dropout : float
Probability of keeping a neuron during dropout phase
network : str
Neural network architecture (*e.g.* `simple`, `vgg`, `inception`)
learning_rate : float
Starting learning rate
learning_rate_decay : float
Learning rate decay
Returns
-------
dict
Dictionary that summarizes the instance and the corresponding model
performance (measured by validation accuracy)
"""
if dl_model == "featdet":
net = FeatureDetectionNetwork(
network_name=instance_name,
image_size=image_size,
nb_channels=3,
nb_labels=nb_labels,
architecture=network,
)
loss_function = "binary_crossentropy"
elif dl_model == "semseg":
net = SemanticSegmentationNetwork(
network_name=instance_name,
image_size=image_size,
nb_channels=3,
nb_labels=nb_labels,
architecture=network,
)
loss_function = "categorical_crossentropy"
else:
logger.error(("Unrecognized model: %s. Please choose amongst %s",
dl_model, AVAILABLE_MODELS))
sys.exit(1)
model = Model(net.X, net.Y)
opt = Adam(lr=learning_rate, decay=learning_rate_decay)
metrics = ["acc", iou, dice_coef]
model.compile(loss=loss_function, optimizer=opt, metrics=metrics)
# Model training
steps = max(nb_training_image // batch_size, 1)
val_steps = max(nb_validation_image // batch_size, 1)
checkpoint_files = [
item for item in os.listdir(output_folder)
if "checkpoint-epoch" in item
]
if len(checkpoint_files) > 0:
model_checkpoint = max(checkpoint_files)
trained_model_epoch = int(model_checkpoint[-5:-3])
checkpoint_complete_path = os.path.join(output_folder,
model_checkpoint)
model.load_weights(checkpoint_complete_path)
logger.info(
"Model weights have been recovered from %s",
checkpoint_complete_path,
)
else:
logger.info(("No available checkpoint for this configuration. "
"The model will be trained from scratch."))
trained_model_epoch = 0
checkpoint_filename = os.path.join(output_folder,
"checkpoint-epoch-{epoch:03d}.h5")
checkpoint = callbacks.ModelCheckpoint(
checkpoint_filename,
monitor="val_loss",
verbose=0,
save_best_only=True,
save_weights_only=False,
mode="auto",
period=1,
)
terminate_on_nan = callbacks.TerminateOnNaN()
earlystop = callbacks.EarlyStopping(monitor="val_loss",
patience=10,
verbose=1,
mode="max")
csv_logger = callbacks.CSVLogger(os.path.join(output_folder,
"training_metrics.csv"),
append=True)
hist = model.fit_generator(
train_generator,
epochs=nb_epochs,
initial_epoch=trained_model_epoch,
steps_per_epoch=steps,
validation_data=validation_generator,
validation_steps=val_steps,
callbacks=[checkpoint, earlystop, terminate_on_nan, csv_logger],
)
ref_metric = max(hist.history.get("val_acc", [np.nan]))
return {
"model": model,
"val_acc": ref_metric,
"batch_size": batch_size,
"network": network,
"dropout": dropout,
"learning_rate": learning_rate,
"learning_rate_decay": learning_rate_decay,
}
args.image_size,
args.batch_size,
label_ids,
inference=True,
seed=SEED)
else:
utils.logger.error(("There is no valid data with the specified parameters. "
"Please generate a valid dataset before calling the training program."))
sys.exit(1)
nb_labels = len(label_ids)
# Model creation
if args.model == "feature_detection":
net = FeatureDetectionNetwork(network_name=instance_name,
image_size=args.image_size,
nb_channels=3,
nb_labels=nb_labels,
dropout=args.dropout,
architecture=args.network)
loss_function = "binary_crossentropy"
elif args.model == "semantic_segmentation":
net = SemanticSegmentationNetwork(network_name=instance_name,
image_size=args.image_size,
nb_channels=nb_labels,
nb_labels=nb_labels,
dropout=args.dropout,
architecture=args.network)
loss_function = "categorical_crossentropy"
else:
utils.logger.error(("Unrecognized model. Please enter 'feature_detection' "
"or 'semantic_segmentation'."))
sys.exit(1)
def run_model(train_generator, validation_generator, dl_model, output_folder,
instance_name, image_size, aggregate_value, nb_labels, nb_epochs,
nb_training_image, nb_validation_image, batch_size, dropout,
network, learning_rate, learning_rate_decay):
"""Run deep learning `dl_model` starting from training and validation data generators, depending on a
range of hyperparameters
Parameters
----------
train_generator : generator
Training data generator
validation_generator : generator
Validation data generator
dl_model : str
Name of the addressed research problem (*e.g.* `feature_detection` or `semantic_segmentation`)
output_folder : str
Name of the folder where the trained model will be stored on the file system
instance_name : str
Name of the instance
image_size : int
Size of images, in pixel (height=width)
aggregate_value : str
Label aggregation policy (either `full` or `aggregated`)
nb_labels : int
Number of labels into the dataset
nb_epochs : int
Number of epochs during which models will be trained
nb_training_image : int
Number of images into the training dataset
nb_validation_image : int
Number of images into the validation dataset
batch_size : int
Number of images into each batch
dropout : float
Probability of keeping a neuron during dropout phase
network : str
Neural network architecture (*e.g.* `simple`, `vgg`, `inception`)
learning_rate : float
Starting learning rate
learning_rate_decay : float
Learning rate decay
Returns
-------
dict
Dictionary that summarizes the instance and the corresponding model performance (measured
by validation accuracy)
"""
if dl_model == "feature_detection":
net = FeatureDetectionNetwork(network_name=instance_name,
image_size=image_size,
nb_channels=3,
nb_labels=nb_labels,
architecture=network)
loss_function = "binary_crossentropy"
elif dl_model == "semantic_segmentation":
net = SemanticSegmentationNetwork(network_name=instance_name,
image_size=image_size,
nb_channels=3,
nb_labels=nb_labels,
architecture=network)
loss_function = "categorical_crossentropy"
else:
utils.logger.error(
("Unrecognized model. Please enter 'feature_detection' "
"or 'semantic_segmentation'."))
sys.exit(1)
model = Model(net.X, net.Y)
opt = Adam(lr=learning_rate, decay=learning_rate_decay)
metrics = ['acc', iou, dice_coef]
model.compile(loss=loss_function, optimizer=opt, metrics=metrics)
# Model training
steps = nb_training_image // batch_size
val_steps = nb_validation_image // batch_size
checkpoint_files = [
item for item in os.listdir(output_folder)
if os.path.isfile(os.path.join(output_folder, item))
]
if len(checkpoint_files) > 0:
model_checkpoint = max(checkpoint_files)
trained_model_epoch = int(model_checkpoint[-5:-3])
checkpoint_complete_path = os.path.join(output_folder,
model_checkpoint)
model.load_weights(checkpoint_complete_path)
utils.logger.info(("Model weights have been recovered from {}"
"").format(checkpoint_complete_path))
else:
utils.logger.info(("No available checkpoint for this configuration. "
"The model will be trained from scratch."))
trained_model_epoch = 0
checkpoint_filename = os.path.join(output_folder,
"checkpoint-epoch-{epoch:03d}.h5")
checkpoint = callbacks.ModelCheckpoint(checkpoint_filename,
monitor='val_loss',
verbose=0,
save_best_only=True,
save_weights_only=False,
mode='auto',
period=1)
terminate_on_nan = callbacks.TerminateOnNaN()
earlystop = callbacks.EarlyStopping(monitor='val_acc',
min_delta=0.001,
patience=10,
verbose=1,
mode='max')
csv_logger = callbacks.CSVLogger(
os.path.join(output_folder, 'training_metrics.csv'))
hist = model.fit_generator(
train_generator,
epochs=nb_epochs,
initial_epoch=trained_model_epoch,
steps_per_epoch=steps,
validation_data=validation_generator,
validation_steps=val_steps,
callbacks=[checkpoint, earlystop, terminate_on_nan, csv_logger])
ref_metric = max(hist.history.get("val_acc", [np.nan]))
return {
'model': model,
'val_acc': ref_metric,
'batch_size': batch_size,
'network': network,
'dropout': dropout,
'learning_rate': learning_rate,
'learning_rate_decay': learning_rate_decay
}
