def test_download_checkpoints(test_output_dirs: TestOutputDirectories, is_ensemble: bool,
runner_config: AzureConfig) -> None:
output_dir = Path(test_output_dirs.root_dir)
assert get_results_blob_path("some_run_id") == "azureml/ExperimentRun/dcid.some_run_id"
# Any recent run ID from a PR build will do. Use a PR build because the checkpoint files are small there.
config = SegmentationModelBase(should_validate=False)
config.set_output_to(output_dir)
runner_config.run_recovery_id = DEFAULT_ENSEMBLE_RUN_RECOVERY_ID if is_ensemble else DEFAULT_RUN_RECOVERY_ID
run_recovery = RunRecovery.download_checkpoints_from_recovery_run(runner_config, config)
run_to_recover = fetch_run(workspace=runner_config.get_workspace(), run_recovery_id=runner_config.run_recovery_id)
expected_checkpoint_file = "1" + CHECKPOINT_FILE_SUFFIX
if is_ensemble:
child_runs = fetch_child_runs(run_to_recover)
expected_files = [Path(config.checkpoint_folder) / run_to_recover.id
/ str(x.get_tags()['cross_validation_split_index']) / expected_checkpoint_file
for x in child_runs]
else:
expected_files = [Path(config.checkpoint_folder) / run_to_recover.id / expected_checkpoint_file]
checkpoint_paths = run_recovery.get_checkpoint_paths(1)
if is_ensemble:
assert len(run_recovery.checkpoints_roots) == len(expected_files)
assert all([(x in [y.parent for y in expected_files]) for x in run_recovery.checkpoints_roots])
assert len(checkpoint_paths) == len(expected_files)
assert all([x in expected_files for x in checkpoint_paths])
else:
assert len(checkpoint_paths) == 1
assert checkpoint_paths[0] == expected_files[0]
assert all([expected_file.exists() for expected_file in expected_files])
def test_copy_child_paths_to_folder(is_ensemble: bool,
extra_code_directory: str,
test_output_dirs: OutputFolderForTests) -> None:
azure_config = AzureConfig(extra_code_directory=extra_code_directory)
fake_model = SegmentationModelBase(should_validate=False)
fake_model.set_output_to(test_output_dirs.root_dir)
# To simulate ensemble models, there are two checkpoints, one in the root dir and one in a folder
checkpoints_absolute, checkpoints_relative = create_checkpoints(fake_model, is_ensemble)
# Simulate a project root: We can't derive that from the repository root because that might point
# into Python's package folder
project_root = Path(__file__).parent.parent
ml_runner = MLRunner(model_config=fake_model, azure_config=azure_config, project_root=project_root)
model_folder = test_output_dirs.root_dir / "final"
ml_runner.copy_child_paths_to_folder(model_folder=model_folder, checkpoint_paths=checkpoints_absolute)
expected_files = [
fixed_paths.ENVIRONMENT_YAML_FILE_NAME,
fixed_paths.MODEL_INFERENCE_JSON_FILE_NAME,
"InnerEye/ML/runner.py",
"InnerEye/ML/model_testing.py",
"InnerEye/Common/fixed_paths.py",
"InnerEye/Common/common_util.py",
]
for r in checkpoints_relative:
expected_files.append(f"{CHECKPOINT_FOLDER}/{r}")
for expected_file in expected_files:
assert (model_folder / expected_file).is_file(), f"File missing: {expected_file}"
trm = model_folder / "TestsOutsidePackage/test_register_model.py"
if extra_code_directory:
assert trm.is_file()
else:
assert not trm.is_file()
def test_download_checkpoints_hyperdrive_run(test_output_dirs: TestOutputDirectories,
runner_config: AzureConfig) -> None:
output_dir = Path(test_output_dirs.root_dir)
config = SegmentationModelBase(should_validate=False)
config.set_output_to(output_dir)
runner_config.run_recovery_id = DEFAULT_ENSEMBLE_RUN_RECOVERY_ID
child_runs = fetch_child_runs(run=fetch_run(runner_config.get_workspace(), DEFAULT_ENSEMBLE_RUN_RECOVERY_ID))
# recover child runs separately also to test hyperdrive child run recovery functionality
expected_checkpoint_file = "1" + CHECKPOINT_FILE_SUFFIX
for child in child_runs:
expected_files = [Path(config.checkpoint_folder) / child.id / expected_checkpoint_file]
run_recovery = RunRecovery.download_checkpoints_from_recovery_run(runner_config, config, child)
assert all([x in expected_files for x in run_recovery.get_checkpoint_paths(epoch=1)])
assert all([expected_file.exists() for expected_file in expected_files])
def test_save_dataset_example(test_output_dirs: OutputFolderForTests) -> None:
"""
Test if the example dataset can be saved as expected.
"""
image_size = (10, 20, 30)
label_size = (2, ) + image_size
spacing = (1, 2, 3)
np.random.seed(0)
# Image should look similar to what a photonormalized image looks like: Centered around 0
image = np.random.rand(*image_size) * 2 - 1
# Labels are expected in one-hot encoding, predictions as class index
labels = np.zeros(label_size, dtype=int)
labels[0] = 1
labels[0, 5:6, 10:11, 15:16] = 0
labels[1, 5:6, 10:11, 15:16] = 1
prediction = np.zeros(image_size, dtype=int)
prediction[4:7, 9:12, 14:17] = 1
dataset_sample = DatasetExample(epoch=1,
patient_id=2,
header=ImageHeader(origin=(0, 1, 0),
direction=(1, 0, 0, 0,
1, 0, 0, 0,
1),
spacing=spacing),
image=image,
prediction=prediction,
labels=labels)
images_folder = test_output_dirs.root_dir
config = SegmentationModelBase(
should_validate=False,
norm_method=PhotometricNormalizationMethod.Unchanged)
config.set_output_to(images_folder)
store_and_upload_example(dataset_sample, config)
image_from_disk = io_util.load_nifti_image(
os.path.join(config.example_images_folder, "p2_e_1_image.nii.gz"))
labels_from_disk = io_util.load_nifti_image(
os.path.join(config.example_images_folder, "p2_e_1_label.nii.gz"))
prediction_from_disk = io_util.load_nifti_image(
os.path.join(config.example_images_folder, "p2_e_1_prediction.nii.gz"))
assert image_from_disk.header.spacing == spacing
# When no photometric normalization is provided when saving, image is multiplied by 1000.
# It is then rounded to int64, but converted back to float when read back in.
expected_from_disk = (image * 1000).astype(np.int16).astype(np.float64)
assert np.array_equal(image_from_disk.image, expected_from_disk)
assert labels_from_disk.header.spacing == spacing
assert np.array_equal(labels_from_disk.image, np.argmax(labels, axis=0))
assert prediction_from_disk.header.spacing == spacing
assert np.array_equal(prediction_from_disk.image, prediction)
def test_invalid_stride_size(test_output_dirs: OutputFolderForTests) -> None:
config = SegmentationModelBase(
architecture="UNet3D",
feature_channels=[1],
crop_size=(64, 64, 64),
test_crop_size=(80, 80, 80),
image_channels=["mr"],
ground_truth_ids=["tumour_mass", "subtract"],
train_batch_size=8,
inference_batch_size=1,
inference_stride_size=(120, 120, 120),
should_validate=False
)
config.set_output_to(test_output_dirs.root_dir)
checkpoint_path = test_output_dirs.root_dir / "checkpoint.ckpt"
create_model_and_store_checkpoint(config, checkpoint_path)
with pytest.raises(ValueError) as ex:
load_from_checkpoint_and_adjust_for_inference(config=config, checkpoint_path=checkpoint_path)
assert "The inference stride size (120, 120, 120) must be smaller" in ex.value.args[0]
assert str(config.inference_stride_size) in ex.value.args[0]
assert str(config.test_crop_size) in ex.value.args[0]
def test_compare_scores_against_baselines_throws(model_proc_split_infer: Tuple[
ModelProcessing, int,
Optional[bool]], test_output_dirs: OutputFolderForTests,
caplog: Any) -> None:
"""
Test that exceptions are raised if the files necessary for baseline comparison are missing,
then test that when all required files are present that baseline comparison files are written.
:param model_proc: Model processing to test.
:param test_output_dirs: Test output directories.
:param caplog: Pytest capture logger.
:return: None.
"""
(model_proc, number_of_cross_validation_splits,
inference_on_test_set) = model_proc_split_infer
config = SegmentationModelBase(
should_validate=False,
comparison_blob_storage_paths=[
('Single', 'dummy_blob_single/outputs/epoch_120/Test'),
('5fold', 'dummy_blob_ensemble/outputs/epoch_120/Test')
],
number_of_cross_validation_splits=number_of_cross_validation_splits)
config.set_output_to(test_output_dirs.root_dir)
azure_config = get_default_azure_config()
# Disable inference
config.inference_on_test_set = False
config.ensemble_inference_on_test_set = False
with caplog.at_level(logging.INFO):
# With no inference output, test that a warning is written to logging
compare_scores_against_baselines(model_config=config,
azure_config=azure_config,
model_proc=model_proc)
assert INFERENCE_DISABLED_WARNING in caplog.text
# Reset inference
config.inference_on_test_set = inference_on_test_set
config.ensemble_inference_on_test_set = None
# If the BEST_EPOCH_FOLDER_NAME folder is missing, expect an exception to be raised.
with pytest.raises(FileNotFoundError) as ex:
compare_scores_against_baselines(model_config=config,
azure_config=azure_config,
model_proc=model_proc)
assert "Cannot compare scores against baselines: no best epoch results found at" in str(
ex)
best_epoch_folder_path = config.outputs_folder
if model_proc == ModelProcessing.ENSEMBLE_CREATION:
best_epoch_folder_path = best_epoch_folder_path / OTHER_RUNS_SUBDIR_NAME / ENSEMBLE_SPLIT_NAME
best_epoch_folder_path = best_epoch_folder_path / BEST_EPOCH_FOLDER_NAME / ModelExecutionMode.TEST.value
best_epoch_folder_path.mkdir(parents=True)
# If the BEST_EPOCH_FOLDER_NAME folder exists but DATASET_CSV_FILE_NAME is missing,
# expect an exception to be raised.
with pytest.raises(FileNotFoundError) as ex:
compare_scores_against_baselines(model_config=config,
azure_config=azure_config,
model_proc=model_proc)
assert "Not comparing with baselines because no " in str(ex)
assert DATASET_CSV_FILE_NAME in str(ex)
model_dataset_path = best_epoch_folder_path / DATASET_CSV_FILE_NAME
dataset_df = create_dataset_df()
dataset_df.to_csv(model_dataset_path)
# If the BEST_EPOCH_FOLDER_NAME folder exists but SUBJECT_METRICS_FILE_NAME is missing,
# expect an exception to be raised.
with pytest.raises(FileNotFoundError) as ex:
compare_scores_against_baselines(model_config=config,
azure_config=azure_config,
model_proc=model_proc)
assert "Not comparing with baselines because no " in str(ex)
assert SUBJECT_METRICS_FILE_NAME in str(ex)
model_metrics_path = best_epoch_folder_path / SUBJECT_METRICS_FILE_NAME
metrics_df = create_metrics_df()
metrics_df.to_csv(model_metrics_path)
baseline = create_comparison_baseline(dataset_df)
# Patch get_comparison_baselines to return the baseline above.
with mock.patch('InnerEye.ML.baselines_util.get_comparison_baselines',
return_value=[baseline]):
compare_scores_against_baselines(model_config=config,
azure_config=azure_config,
model_proc=model_proc)
# Check the wilcoxoon results file is present and has expected contents.
wilcoxon_path = best_epoch_folder_path / BASELINE_WILCOXON_RESULTS_FILE
assert wilcoxon_path.is_file()
wilcoxon_lines = [
line.strip() for line in wilcoxon_path.read_text().splitlines()
]
check_wilcoxon_lines(wilcoxon_lines, baseline)
# Check the full metrics results file is present.
full_metrics_path = best_epoch_folder_path / FULL_METRICS_DATAFRAME_FILE
assert full_metrics_path.is_file()
