def test_model_backup_loading(shapes_image_size, shapes_sample_config, shapes_temp_dir): """Test the model checkpoint recovering """ config = read_config(shapes_sample_config) label_ids = [x['id'] for x in config['labels'] if x['is_evaluate']] cnn = FeatureDetectionNetwork("test", image_size=shapes_image_size, nb_labels=len(label_ids)) model = Model(cnn.X, cnn.Y) old_weights = model.get_weights() checkpoint_path = os.path.join(str(shapes_temp_dir), "checkpoints") if os.path.isdir(checkpoint_path): checkpoints = os.listdir(checkpoint_path) if len(checkpoints) > 0: model_checkpoint = max(checkpoints) trained_model_epoch = int(model_checkpoint[-5:-3]) checkpoint_complete_path = os.path.join(checkpoint_path, model_checkpoint) model.load_weights(checkpoint_complete_path) else: trained_model_epoch = 0 new_weights = model.get_weights() assert trained_model_epoch > 0 assert len(old_weights) == len(new_weights) assert old_weights[0].shape == new_weights[0].shape # Test if old and new weights are different (at least for one layer) assert any(not np.allclose(lhs, rhs) for lhs, rhs in zip(old_weights, new_weights))
def init_model( problem, instance_name, image_size, nb_labels, dropout, network ): """Initialize a convolutional neural network with Keras API starting from a set of parameters Parameters ---------- problem : str Type of solved problem, either `feature_detection` or `semantic_segmentation` instance_name : str Name of the instance, for identification purpose image_size : int Image size, in pixels (height=width) nb_labels : int Number of output labels dropout : float Dropout rate network : str Network architecture Returns ------- keras.models.Model Convolutional neural network """ K.clear_session() if problem == "feature_detection": net = FeatureDetectionNetwork( network_name=instance_name, image_size=image_size, nb_labels=nb_labels, dropout=dropout, architecture=network, ) elif problem == "semantic_segmentation": net = SemanticSegmentationNetwork( network_name=instance_name, image_size=image_size, nb_labels=nb_labels, dropout=dropout, architecture=network, ) else: logger.error( ( "Unrecognized model. Please enter 'feature_detection' " "or 'semantic_segmentation'." ) ) sys.exit(1) return Model(net.X, net.Y)
def test_simple_network_architecture(shapes_image_size, nb_channels, shapes_nb_labels): """Test a simple feature detection network """ net = FeatureDetectionNetwork("fd_test", image_size=shapes_image_size, nb_channels=nb_channels, nb_labels=shapes_nb_labels) m = Model(net.X, net.Y) input_shape = m.input_shape output_shape = m.output_shape assert len(input_shape) == 4 assert input_shape[1:] == (shapes_image_size, shapes_image_size, nb_channels) assert len(output_shape) == 2 assert output_shape[1] == shapes_nb_labels
def test_model_training( shapes_image_size, shapes_sample, shapes_sample_config, shapes_temp_dir, shapes_nb_images, ): """Test the training of a simple neural network with Keras API, as well as model inference and trained model backup One big test function to avoid duplicating the training operations (that can be long) """ BATCH_SIZE = 10 NB_EPOCHS = 1 NB_STEPS = shapes_nb_images // BATCH_SIZE config = read_config(shapes_sample_config) label_ids = [x["id"] for x in config["labels"] if x["is_evaluate"]] gen = create_generator( "shapes", "featdet", shapes_sample, shapes_image_size, BATCH_SIZE, config["labels"], ) cnn = FeatureDetectionNetwork("test", image_size=shapes_image_size, nb_labels=len(label_ids)) model = Model(cnn.X, cnn.Y) model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["acc"]) hist = model.fit_generator(gen, epochs=NB_EPOCHS, steps_per_epoch=NB_STEPS) assert len(hist.history) == 2 assert all(k in hist.history.keys() for k in ["acc", "loss"]) assert hist.history["acc"][0] >= 0 and hist.history["acc"][0] <= 1 test_image = np.random.randint( 0, 255, [BATCH_SIZE, shapes_image_size, shapes_image_size, 3]) score = model.predict(test_image) assert score.shape == (BATCH_SIZE, len(label_ids)) assert all(0 <= s and s <= 1 for s in score.ravel()) BACKUP_FILENAME = os.path.join( str(shapes_temp_dir), "checkpoints", "test_model_{:02d}.h5".format(NB_EPOCHS), ) model.save(BACKUP_FILENAME) assert os.path.isfile(BACKUP_FILENAME)
def test_inception_network_architecture(mapillary_image_size, nb_channels, mapillary_nb_labels): """Test a Inception-inspired feature detection network """ net = FeatureDetectionNetwork("fd_test", image_size=mapillary_image_size, nb_channels=nb_channels, nb_labels=mapillary_nb_labels, architecture="inception") m = Model(net.X, net.Y) input_shape = m.input_shape output_shape = m.output_shape assert len(input_shape) == 4 assert input_shape[1:] == (mapillary_image_size, mapillary_image_size, nb_channels) assert len(output_shape) == 2 assert output_shape[1] == mapillary_nb_labels
def run_model( train_generator, validation_generator, dl_model, output_folder, instance_name, image_size, nb_labels, nb_epochs, nb_training_image, nb_validation_image, batch_size, dropout, network, learning_rate, learning_rate_decay, ): """Run deep learning `dl_model` starting from training and validation data generators, depending on a range of hyperparameters Parameters ---------- train_generator : generator Training data generator validation_generator : generator Validation data generator dl_model : str Name of the addressed research problem (*e.g.* `feature_detection` or `semantic_segmentation`) output_folder : str Name of the folder where the trained model will be stored on the file system instance_name : str Name of the instance image_size : int Size of images, in pixel (height=width) nb_labels : int Number of labels into the dataset nb_epochs : int Number of epochs during which models will be trained nb_training_image : int Number of images into the training dataset nb_validation_image : int Number of images into the validation dataset batch_size : int Number of images into each batch dropout : float Probability of keeping a neuron during dropout phase network : str Neural network architecture (*e.g.* `simple`, `vgg`, `inception`) learning_rate : float Starting learning rate learning_rate_decay : float Learning rate decay Returns ------- dict Dictionary that summarizes the instance and the corresponding model performance (measured by validation accuracy) """ if dl_model == "featdet": net = FeatureDetectionNetwork( network_name=instance_name, image_size=image_size, nb_channels=3, nb_labels=nb_labels, architecture=network, ) loss_function = "binary_crossentropy" elif dl_model == "semseg": net = SemanticSegmentationNetwork( network_name=instance_name, image_size=image_size, nb_channels=3, nb_labels=nb_labels, architecture=network, ) loss_function = "categorical_crossentropy" else: logger.error(("Unrecognized model: %s. Please choose amongst %s", dl_model, AVAILABLE_MODELS)) sys.exit(1) model = Model(net.X, net.Y) opt = Adam(lr=learning_rate, decay=learning_rate_decay) metrics = ["acc", iou, dice_coef] model.compile(loss=loss_function, optimizer=opt, metrics=metrics) # Model training steps = max(nb_training_image // batch_size, 1) val_steps = max(nb_validation_image // batch_size, 1) checkpoint_files = [ item for item in os.listdir(output_folder) if "checkpoint-epoch" in item ] if len(checkpoint_files) > 0: model_checkpoint = max(checkpoint_files) trained_model_epoch = int(model_checkpoint[-5:-3]) checkpoint_complete_path = os.path.join(output_folder, model_checkpoint) model.load_weights(checkpoint_complete_path) logger.info( "Model weights have been recovered from %s", checkpoint_complete_path, ) else: logger.info(("No available checkpoint for this configuration. " "The model will be trained from scratch.")) trained_model_epoch = 0 checkpoint_filename = os.path.join(output_folder, "checkpoint-epoch-{epoch:03d}.h5") checkpoint = callbacks.ModelCheckpoint( checkpoint_filename, monitor="val_loss", verbose=0, save_best_only=True, save_weights_only=False, mode="auto", period=1, ) terminate_on_nan = callbacks.TerminateOnNaN() earlystop = callbacks.EarlyStopping(monitor="val_loss", patience=10, verbose=1, mode="max") csv_logger = callbacks.CSVLogger(os.path.join(output_folder, "training_metrics.csv"), append=True) hist = model.fit_generator( train_generator, epochs=nb_epochs, initial_epoch=trained_model_epoch, steps_per_epoch=steps, validation_data=validation_generator, validation_steps=val_steps, callbacks=[checkpoint, earlystop, terminate_on_nan, csv_logger], ) ref_metric = max(hist.history.get("val_acc", [np.nan])) return { "model": model, "val_acc": ref_metric, "batch_size": batch_size, "network": network, "dropout": dropout, "learning_rate": learning_rate, "learning_rate_decay": learning_rate_decay, }
args.image_size, args.batch_size, label_ids, inference=True, seed=SEED) else: utils.logger.error(("There is no valid data with the specified parameters. " "Please generate a valid dataset before calling the training program.")) sys.exit(1) nb_labels = len(label_ids) # Model creation if args.model == "feature_detection": net = FeatureDetectionNetwork(network_name=instance_name, image_size=args.image_size, nb_channels=3, nb_labels=nb_labels, dropout=args.dropout, architecture=args.network) loss_function = "binary_crossentropy" elif args.model == "semantic_segmentation": net = SemanticSegmentationNetwork(network_name=instance_name, image_size=args.image_size, nb_channels=nb_labels, nb_labels=nb_labels, dropout=args.dropout, architecture=args.network) loss_function = "categorical_crossentropy" else: utils.logger.error(("Unrecognized model. Please enter 'feature_detection' " "or 'semantic_segmentation'.")) sys.exit(1)
def run_model(train_generator, validation_generator, dl_model, output_folder, instance_name, image_size, aggregate_value, nb_labels, nb_epochs, nb_training_image, nb_validation_image, batch_size, dropout, network, learning_rate, learning_rate_decay): """Run deep learning `dl_model` starting from training and validation data generators, depending on a range of hyperparameters Parameters ---------- train_generator : generator Training data generator validation_generator : generator Validation data generator dl_model : str Name of the addressed research problem (*e.g.* `feature_detection` or `semantic_segmentation`) output_folder : str Name of the folder where the trained model will be stored on the file system instance_name : str Name of the instance image_size : int Size of images, in pixel (height=width) aggregate_value : str Label aggregation policy (either `full` or `aggregated`) nb_labels : int Number of labels into the dataset nb_epochs : int Number of epochs during which models will be trained nb_training_image : int Number of images into the training dataset nb_validation_image : int Number of images into the validation dataset batch_size : int Number of images into each batch dropout : float Probability of keeping a neuron during dropout phase network : str Neural network architecture (*e.g.* `simple`, `vgg`, `inception`) learning_rate : float Starting learning rate learning_rate_decay : float Learning rate decay Returns ------- dict Dictionary that summarizes the instance and the corresponding model performance (measured by validation accuracy) """ if dl_model == "feature_detection": net = FeatureDetectionNetwork(network_name=instance_name, image_size=image_size, nb_channels=3, nb_labels=nb_labels, architecture=network) loss_function = "binary_crossentropy" elif dl_model == "semantic_segmentation": net = SemanticSegmentationNetwork(network_name=instance_name, image_size=image_size, nb_channels=3, nb_labels=nb_labels, architecture=network) loss_function = "categorical_crossentropy" else: utils.logger.error( ("Unrecognized model. Please enter 'feature_detection' " "or 'semantic_segmentation'.")) sys.exit(1) model = Model(net.X, net.Y) opt = Adam(lr=learning_rate, decay=learning_rate_decay) metrics = ['acc', iou, dice_coef] model.compile(loss=loss_function, optimizer=opt, metrics=metrics) # Model training steps = nb_training_image // batch_size val_steps = nb_validation_image // batch_size checkpoint_files = [ item for item in os.listdir(output_folder) if os.path.isfile(os.path.join(output_folder, item)) ] if len(checkpoint_files) > 0: model_checkpoint = max(checkpoint_files) trained_model_epoch = int(model_checkpoint[-5:-3]) checkpoint_complete_path = os.path.join(output_folder, model_checkpoint) model.load_weights(checkpoint_complete_path) utils.logger.info(("Model weights have been recovered from {}" "").format(checkpoint_complete_path)) else: utils.logger.info(("No available checkpoint for this configuration. " "The model will be trained from scratch.")) trained_model_epoch = 0 checkpoint_filename = os.path.join(output_folder, "checkpoint-epoch-{epoch:03d}.h5") checkpoint = callbacks.ModelCheckpoint(checkpoint_filename, monitor='val_loss', verbose=0, save_best_only=True, save_weights_only=False, mode='auto', period=1) terminate_on_nan = callbacks.TerminateOnNaN() earlystop = callbacks.EarlyStopping(monitor='val_acc', min_delta=0.001, patience=10, verbose=1, mode='max') csv_logger = callbacks.CSVLogger( os.path.join(output_folder, 'training_metrics.csv')) hist = model.fit_generator( train_generator, epochs=nb_epochs, initial_epoch=trained_model_epoch, steps_per_epoch=steps, validation_data=validation_generator, validation_steps=val_steps, callbacks=[checkpoint, earlystop, terminate_on_nan, csv_logger]) ref_metric = max(hist.history.get("val_acc", [np.nan])) return { 'model': model, 'val_acc': ref_metric, 'batch_size': batch_size, 'network': network, 'dropout': dropout, 'learning_rate': learning_rate, 'learning_rate_decay': learning_rate_decay }