def test_non_image_encoder(
test_output_dirs: OutputFolderForTests,
hidden_layer_num_feature_channels: Optional[int]) -> None:
"""
Test if we can build a simple MLP model that only feeds off non-image features.
"""
dataset_folder = Path(test_output_dirs.make_sub_dir("dataset"))
dataset_contents = _get_fake_dataset_contents()
(dataset_folder / DATASET_CSV_FILE_NAME).write_text(dataset_contents)
config = NonImageEncoder(
should_validate=False,
hidden_layer_num_feature_channels=hidden_layer_num_feature_channels)
config.local_dataset = dataset_folder
config.set_output_to(test_output_dirs.root_dir)
config.max_batch_grad_cam = 1
config.validate()
# run model training
_, checkpoint_handler = model_train_unittest(
config, output_folder=test_output_dirs)
# run model inference
runner = MLRunner(config)
runner.setup()
runner.model_inference_train_and_test(
checkpoint_paths=checkpoint_handler.get_checkpoints_to_test())
assert config.get_total_number_of_non_imaging_features() == 18
def run_model_inference_train_and_test(
test_output_dirs: OutputFolderForTests,
perform_cross_validation: bool,
inference_on_train_set: Optional[bool] = None,
inference_on_val_set: Optional[bool] = None,
inference_on_test_set: Optional[bool] = None,
ensemble_inference_on_train_set: Optional[bool] = None,
ensemble_inference_on_val_set: Optional[bool] = None,
ensemble_inference_on_test_set: Optional[bool] = None,
model_proc: ModelProcessing = ModelProcessing.DEFAULT) -> None:
"""
Test running inference produces expected output metrics, files, folders and calls to upload_folder.
:param test_output_dirs: Test output directories.
:param perform_cross_validation: Whether to test with cross validation.
:param inference_on_train_set: Override for inference on train data sets.
:param inference_on_val_set: Override for inference on validation data sets.
:param inference_on_test_set: Override for inference on test data sets.
:param ensemble_inference_on_train_set: Override for ensemble inference on train data sets.
:param ensemble_inference_on_val_set: Override for ensemble inference on validation data sets.
:param ensemble_inference_on_test_set: Override for ensemble inference on test data sets.
:param model_proc: Model processing to test.
:return: None.
"""
dummy_model = DummyModel()
config = PassThroughModel()
# Copy settings from DummyModel
config.image_channels = dummy_model.image_channels
config.ground_truth_ids = dummy_model.ground_truth_ids
config.ground_truth_ids_display_names = dummy_model.ground_truth_ids_display_names
config.colours = dummy_model.colours
config.fill_holes = dummy_model.fill_holes
config.roi_interpreted_types = dummy_model.roi_interpreted_types
config.test_crop_size = (16, 16, 16)
config.number_of_cross_validation_splits = 2 if perform_cross_validation else 0
config.inference_on_train_set = inference_on_train_set
config.inference_on_val_set = inference_on_val_set
config.inference_on_test_set = inference_on_test_set
config.ensemble_inference_on_train_set = ensemble_inference_on_train_set
config.ensemble_inference_on_val_set = ensemble_inference_on_val_set
config.ensemble_inference_on_test_set = ensemble_inference_on_test_set
# Plotting crashes with random TCL errors on Windows, disable that for Windows PR builds.
config.is_plotting_enabled = common_util.is_linux()
config.set_output_to(test_output_dirs.root_dir)
train_and_test_data_small_dir = test_output_dirs.root_dir / "train_and_test_data_small"
config.local_dataset = create_train_and_test_data_small_dataset(
config.test_crop_size, full_ml_test_data_path(), "train_and_test_data",
train_and_test_data_small_dir, "data")
checkpoint_path = config.checkpoint_folder / LAST_CHECKPOINT_FILE_NAME_WITH_SUFFIX
create_model_and_store_checkpoint(config, checkpoint_path)
checkpoint_handler = get_default_checkpoint_handler(
model_config=config, project_root=test_output_dirs.root_dir)
checkpoint_handler.additional_training_done()
mock_upload_path = test_output_dirs.root_dir / "mock_upload"
mock_upload_path.mkdir()
run = create_mock_run(mock_upload_path, config)
azure_config = Mock(name='mock_azure_config')
azure_config.fetch_run.return_value = run
runner = MLRunner(model_config=config, azure_config=azure_config)
with mock.patch("InnerEye.ML.model_testing.PARENT_RUN_CONTEXT", run):
metrics = runner.model_inference_train_and_test(
checkpoint_paths=checkpoint_handler.get_checkpoints_to_test(),
model_proc=model_proc)
if model_proc == ModelProcessing.ENSEMBLE_CREATION:
# Create a fake ensemble dataset.csv
dataset_df = create_dataset_df()
dataset_df.to_csv(config.outputs_folder / DATASET_CSV_FILE_NAME)
with mock.patch.object(PlotCrossValidationConfig,
'azure_config',
return_value=azure_config):
with mock.patch("InnerEye.Azure.azure_util.PARENT_RUN_CONTEXT",
run):
with mock.patch("InnerEye.ML.run_ml.PARENT_RUN_CONTEXT", run):
runner.plot_cross_validation_and_upload_results()
runner.generate_report(ModelProcessing.ENSEMBLE_CREATION)
if model_proc == ModelProcessing.DEFAULT:
named_metrics = {
ModelExecutionMode.TRAIN: inference_on_train_set,
ModelExecutionMode.TEST: inference_on_test_set,
ModelExecutionMode.VAL: inference_on_val_set
}
else:
named_metrics = {
ModelExecutionMode.TRAIN: ensemble_inference_on_train_set,
ModelExecutionMode.TEST: ensemble_inference_on_test_set,
ModelExecutionMode.VAL: ensemble_inference_on_val_set
}
error = ''
expected_upload_folder_count = 0
for mode, flag in named_metrics.items():
if mode in metrics:
metric = metrics[mode]
assert isinstance(metric, InferenceMetricsForSegmentation)
if flag is None:
# No override supplied, calculate the expected default:
if model_proc == ModelProcessing.DEFAULT:
if not perform_cross_validation:
# If a "normal" run then default to val or test.
flag = mode == ModelExecutionMode.TEST
else:
# If an ensemble child then default to never.
flag = False
else:
# If an ensemble then default to test only.
flag = mode == ModelExecutionMode.TEST
if mode in metrics and not flag:
error = error + f"Error: {mode.value} cannot be not None."
elif mode not in metrics and flag:
error = error + f"Error: {mode.value} cannot be None."
results_folder = config.outputs_folder / get_best_epoch_results_path(
mode, model_proc)
folder_exists = results_folder.is_dir()
assert folder_exists == flag
if flag and model_proc == ModelProcessing.ENSEMBLE_CREATION:
expected_upload_folder_count = expected_upload_folder_count + 1
expected_name = get_best_epoch_results_path(
mode, ModelProcessing.DEFAULT)
run.upload_folder.assert_any_call(name=str(expected_name),
path=str(results_folder))
if len(error):
raise ValueError(error)
if model_proc == ModelProcessing.ENSEMBLE_CREATION:
# The report should have been mock uploaded
expected_upload_folder_count = expected_upload_folder_count + 1
assert run.upload_folder.call_count == expected_upload_folder_count