def test_save_outliers(test_config: PlotCrossValidationConfig,
test_output_dirs: OutputFolderForTests) -> None:
"""Test to make sure the outlier file for a split is as expected"""
test_config.outputs_directory = test_output_dirs.root_dir
test_config.outlier_range = 0
assert test_config.run_recovery_id
dataset_split_metrics = {
x: _get_metrics_df(test_config.run_recovery_id, x)
for x in [ModelExecutionMode.VAL]
}
save_outliers(test_config, dataset_split_metrics,
test_config.outputs_directory)
filename = f"{ModelExecutionMode.VAL.value}_outliers.txt"
assert_text_files_match(full_file=test_config.outputs_directory / filename,
expected_file=full_ml_test_data_path(filename))
# Now test without the CSV_INSTITUTION_HEADER and CSV_SERIES_HEADER columns, which will be missing in institutions' environments
dataset_split_metrics_pruned = {
x: _get_metrics_df(test_config.run_recovery_id, x).drop(
columns=[CSV_INSTITUTION_HEADER, CSV_SERIES_HEADER],
errors="ignore")
for x in [ModelExecutionMode.VAL]
}
save_outliers(test_config, dataset_split_metrics_pruned,
test_config.outputs_directory)
test_data_filename = f"{ModelExecutionMode.VAL.value}_outliers_pruned.txt"
assert_text_files_match(
full_file=test_config.outputs_directory / filename,
expected_file=full_ml_test_data_path(test_data_filename))
def test_create_from_checkpoint_ensemble() -> None:
config = ClassificationModelForTesting()
checkpoint_folder_non_exist = "classification_data_generated_random/checkpoints/non_exist.pth.tar"
path_to_checkpoint_non_exist = full_ml_test_data_path(
checkpoint_folder_non_exist)
checkpoint_folder_exist = "classification_data_generated_random/checkpoints/1_checkpoint.pth.tar"
path_to_checkpoint_exist = full_ml_test_data_path(checkpoint_folder_exist)
# when all checkpoints do not exist, raise error
with pytest.raises(ValueError):
paths_to_checkpoint = [path_to_checkpoint_non_exist] * 5
ScalarEnsemblePipeline.create_from_checkpoint(paths_to_checkpoint,
config)
# when a few checkpoints exist, ensemble with those
paths_to_checkpoint = [path_to_checkpoint_non_exist
] * 3 + [path_to_checkpoint_exist] * 2
inference_pipeline = ScalarEnsemblePipeline.create_from_checkpoint(
paths_to_checkpoint, config)
assert isinstance(inference_pipeline, ScalarEnsemblePipeline)
assert len(inference_pipeline.pipelines) == 2
# when all checkpoints exist
paths_to_checkpoint = [path_to_checkpoint_exist] * 5
inference_pipeline = ScalarEnsemblePipeline.create_from_checkpoint(
paths_to_checkpoint, config)
assert isinstance(inference_pipeline, ScalarEnsemblePipeline)
assert len(inference_pipeline.pipelines) == 5
def test_try_create_model_load_from_checkpoint_and_adjust(config: ModelConfigBase, checkpoint_path: str,
model_execution_mode: ModelExecutionMode) -> None:
config.use_gpu = True
# no checkpoint path provided
model_and_info = ModelAndInfo(config,
model_execution_mode=model_execution_mode,
checkpoint_path=None)
with pytest.raises(ValueError):
model_and_info.model
model_loaded = model_and_info.try_create_model_load_from_checkpoint_and_adjust()
assert model_loaded
assert isinstance(model_and_info.model, DataParallelModel)
# Invalid checkpoint path provided
model_and_info = ModelAndInfo(config,
model_execution_mode=model_execution_mode,
checkpoint_path=full_ml_test_data_path("non_exist.pth.tar"))
model_loaded = model_and_info.try_create_model_load_from_checkpoint_and_adjust()
assert not model_loaded
# Current code assumes that even if this function returns False, the model itself was created, only the checkpoint
# loading failed.
assert isinstance(model_and_info.model, DataParallelModel)
# Valid checkpoint path provided
model_and_info = ModelAndInfo(config,
model_execution_mode=model_execution_mode,
checkpoint_path=full_ml_test_data_path(checkpoint_path))
model_loaded = model_and_info.try_create_model_load_from_checkpoint_and_adjust()
assert model_loaded
assert isinstance(model_and_info.model, DataParallelModel)
assert model_and_info.checkpoint_epoch == 1
def test_get_checkpoints_to_test(
test_output_dirs: OutputFolderForTests) -> None:
config = ModelConfigBase(should_validate=False)
config.set_output_to(test_output_dirs.root_dir)
config.outputs_folder.mkdir()
manage_recovery = get_default_checkpoint_handler(
model_config=config, project_root=test_output_dirs.root_dir)
# Set a local_weights_path to get checkpoint from. Model has not trained and no run recovery provided,
# so the local weights should be used ignoring any epochs to test
config.epochs_to_test = [1, 2]
local_weights_path = test_output_dirs.root_dir / "exist.pth"
stored_checkpoint = create_checkpoint_path(
full_ml_test_data_path("checkpoints"), epoch=1)
shutil.copyfile(str(stored_checkpoint), local_weights_path)
config.local_weights_path = local_weights_path
manage_recovery.discover_and_download_checkpoints_from_previous_runs()
checkpoint_and_paths = manage_recovery.get_checkpoints_to_test()
assert checkpoint_and_paths
assert len(checkpoint_and_paths) == 1
assert checkpoint_and_paths[0].epoch == 0
assert checkpoint_and_paths[0].checkpoint_paths == [
manage_recovery.model_config.outputs_folder / WEIGHTS_FILE
]
# Now set a run recovery object and set the start epoch to 1, so we get one epoch from
# run recovery and one from the training checkpoints
manage_recovery.azure_config.run_recovery_id = DEFAULT_RUN_RECOVERY_ID
config.start_epoch = 1
manage_recovery.additional_training_done()
manage_recovery.discover_and_download_checkpoints_from_previous_runs()
# Copy checkpoint to make it seem like training has happened
stored_checkpoint = create_checkpoint_path(
path=full_ml_test_data_path("checkpoints"), epoch=1)
expected_checkpoint = create_checkpoint_path(path=config.checkpoint_folder,
epoch=2)
shutil.copyfile(str(stored_checkpoint), str(expected_checkpoint))
checkpoint_and_paths = manage_recovery.get_checkpoints_to_test()
assert checkpoint_and_paths
assert len(checkpoint_and_paths) == 2
assert checkpoint_and_paths[0].epoch == 1
assert checkpoint_and_paths[0].checkpoint_paths == [
create_checkpoint_path(path=config.checkpoint_folder /
DEFAULT_RUN_RECOVERY_ID.split(":")[1],
epoch=1)
]
assert checkpoint_and_paths[1].epoch == 2
assert checkpoint_and_paths[1].checkpoint_paths == [
create_checkpoint_path(path=config.checkpoint_folder, epoch=2)
]
# This epoch does not exist
config.epochs_to_test = [3]
checkpoint_and_paths = manage_recovery.get_checkpoints_to_test()
assert checkpoint_and_paths is None
def test_create_summary(test_output_dirs: OutputFolderForTests) -> None:
"""
Test that summaries of CV performance per mode, and per mode per structure, look like they should.
"""
root = test_output_dirs.root_dir
test_file = full_ml_test_data_path("MetricsAcrossAllRuns.csv")
df = pd.read_csv(test_file)
file1, file2 = create_results_breakdown(df, root)
expected1 = full_ml_test_data_path(METRICS_BY_MODE_AND_STRUCTURE_FILE)
expected2 = full_ml_test_data_path(METRICS_BY_MODE_FILE)
assert file1.read_text() == expected1.read_text()
assert file2.read_text() == expected2.read_text()
def test_show_non_square_images(test_output_dirs: OutputFolderForTests) -> None:
input_file = full_ml_test_data_path("patch_sampling") / "scan_small.nii.gz"
input = load_nifti_image(input_file)
image = input.image
shape = image.shape
mask = np.zeros_like(image)
mask[shape[0] // 2, shape[1] // 2, shape[2] // 2] = 1
for dim in range(3):
scan_with_transparent_overlay(image, mask, dim, shape[dim] // 2, spacing=input.header.spacing)
actual_file = Path(test_output_dirs.root_dir) / f"dim_{dim}.png"
resize_and_save(5, 5, actual_file)
expected = full_ml_test_data_path("patch_sampling") / f"overlay_with_aspect_dim{dim}.png"
# To update the stored results, uncomment this line:
# expected.write_bytes(actual_file.read_bytes())
assert_binary_files_match(actual_file, expected)
def load_train_and_test_data_channels(
patient_ids: List[int],
normalization_fn: PhotometricNormalization) -> List[Sample]:
if np.any(np.asarray(patient_ids) <= 0):
raise ValueError("data_items must be >= 0")
file_name = lambda k, y: full_ml_test_data_path("train_and_test_data"
) / f"id{k}_{y}.nii.gz"
get_sample = lambda z: io_util.load_images_from_dataset_source(
dataset_source=PatientDatasetSource(
metadata=PatientMetadata(patient_id=z),
image_channels=[file_name(z, c) for c in TEST_CHANNEL_IDS],
mask_channel=file_name(z, TEST_MASK_ID),
ground_truth_channels=[file_name(z, TEST_GT_ID)]))
samples = []
for x in patient_ids:
sample = get_sample(x)
sample = Sample(image=normalization_fn.transform(
sample.image, sample.mask),
mask=sample.mask,
labels=sample.labels,
metadata=sample.metadata)
samples.append(sample)
return samples
def test_random_gamma_image(image_as_tensor: torch.Tensor) -> None:
human_readable_transformed = to_pil_image(
RandomGamma(scale=(2, 3))(image_as_tensor).squeeze(0))
expected_pil_image = Image.open(
full_ml_test_data_path() /
"gamma_transformed_image_and_contour.png").convert("RGB")
assert expected_pil_image == human_readable_transformed
def __init__(self,
has_local_dataset: bool = False,
has_azure_dataset: bool = False):
super().__init__()
self.local_dataset = full_ml_test_data_path(
"lightning_module_data") if has_local_dataset else None
self.azure_dataset_id = "azure_dataset" if has_azure_dataset else ""
def test_create_from_checkpoint_non_ensemble() -> None:
config = ClassificationModelForTesting()
# when checkpoint does not exist, return None
checkpoint_folder = "classification_data_generated_random/checkpoints/non_exist.pth.tar"
path_to_checkpoint = full_ml_test_data_path(checkpoint_folder)
inference_pipeline = ScalarInferencePipeline.create_from_checkpoint(
path_to_checkpoint, config)
assert inference_pipeline is None
checkpoint_folder = "classification_data_generated_random/checkpoints/1_checkpoint.pth.tar"
path_to_checkpoint = full_ml_test_data_path(checkpoint_folder)
inference_pipeline = ScalarInferencePipeline.create_from_checkpoint(
path_to_checkpoint, config)
assert isinstance(inference_pipeline, ScalarInferencePipeline)
assert inference_pipeline.epoch == 1
def create_checkpoints(model_config: SegmentationModelBase,
is_ensemble: bool) -> Tuple[List[Path], List[Path]]:
"""
Copies 1 or 2 checkpoint files from the stored test data into the model's checkpoint folder, and returns
the absolute paths of those files.
:param model_config: The model configuration, where a correct output folder must be set.
:param is_ensemble: If true, 2 checkpoints (simulating an ensemble run) will be created. If false, only a
single checkpoint will be created.
"""
# To simulate ensemble models, there are two checkpoints, one in the root dir and one in a folder
stored_checkpoints = full_ml_test_data_path('checkpoints')
checkpoints = list(
stored_checkpoints.rglob("*.tar")) if is_ensemble else list(
stored_checkpoints.glob("*.tar"))
assert len(checkpoints) == (2 if is_ensemble else 1)
checkpoints_relative = [
checkpoint.relative_to(stored_checkpoints)
for checkpoint in checkpoints
]
checkpoints_absolute = []
# copy_child_paths_to_folder expects all checkpoints to live inside the model's checkpoint folder
for (file_abs,
file_relative) in list(zip(checkpoints, checkpoints_relative)):
destination_abs = model_config.checkpoint_folder / file_relative
destination_abs.parent.mkdir(exist_ok=True, parents=True)
shutil.copy(str(file_abs), str(destination_abs))
checkpoints_absolute.append(destination_abs)
return checkpoints_absolute, checkpoints_relative
def create_run_result_file_list(config: PlotCrossValidationConfig,
folder: str) -> List[RunResultFiles]:
"""
Creates a list of input files for cross validation analysis, from files stored inside of the test data folder.
:param config: The overall cross validation config
:param folder: The folder to read from, inside of test_data/plot_cross_validation.
:return:
"""
full_folder = full_ml_test_data_path("plot_cross_validation") / folder
files: List[RunResultFiles] = []
previous_dataset_file = None
for split in ["0", "1"]:
for mode in config.execution_modes_to_download():
metrics_file = full_folder / split / mode.value / METRICS_FILE_NAME
dataset_file: Optional[
Path] = full_folder / split / DATASET_CSV_FILE_NAME
if dataset_file.exists(): # type: ignore
# Reduce amount of checked-in large files. dataset files can be large, and usually duplicate across
# runs. Store only a copy in split 0, re-use in split 1.
previous_dataset_file = dataset_file
else:
dataset_file = previous_dataset_file
if metrics_file.exists():
file = RunResultFiles(
execution_mode=mode,
metrics_file=metrics_file,
dataset_csv_file=dataset_file,
run_recovery_id=config.run_recovery_id + "_" +
split, # type: ignore
split_index=split)
files.append(file)
return files
def test_submit_for_inference(test_output_dirs: OutputFolderForTests) -> None:
"""
Execute the submit_for_inference script on the model that was recently trained. This starts an AzureML job,
and downloads the segmentation. Then check if the segmentation was actually produced.
:param test_output_dirs: Test output directories.
"""
model = get_most_recent_model(fallback_run_id_for_local_execution=FALLBACK_SINGLE_RUN)
assert PYTHON_ENVIRONMENT_NAME in model.tags, "Environment name not present in model properties"
# Both parts of this test rely on the same model that was trained in a previous run. If these tests are executed
# independently (via pytest.mark.parametrize), get_most_recent_model would pick up the AML run that the
# previously executed part of this test submitted.
for use_dicom in [False, True]:
if use_dicom:
size = (64, 64, 64)
spacing = (1., 1., 2.5)
image_file = test_output_dirs.root_dir / "temp_pack_dicom_series" / "dicom_series.zip"
scratch_folder = test_output_dirs.root_dir / "temp_dicom_series"
zip_random_dicom_series(size, spacing, image_file, scratch_folder)
else:
image_file = fixed_paths_for_tests.full_ml_test_data_path() / "train_and_test_data" / "id1_channel1.nii.gz"
assert image_file.exists(), f"Image file not found: {image_file}"
settings_file = fixed_paths.SETTINGS_YAML_FILE
assert settings_file.exists(), f"Settings file not found: {settings_file}"
args = ["--image_file", str(image_file),
"--model_id", model.id,
"--settings", str(settings_file),
"--download_folder", str(test_output_dirs.root_dir),
"--cluster", "training-nc12",
"--experiment", get_experiment_name_from_environment() or "model_inference",
"--use_dicom", str(use_dicom)]
download_file = DEFAULT_RESULT_ZIP_DICOM_NAME if use_dicom else DEFAULT_RESULT_IMAGE_NAME
seg_path = test_output_dirs.root_dir / download_file
assert not seg_path.exists(), f"Result file {seg_path} should not yet exist"
submit_for_inference.main(args, project_root=fixed_paths.repository_root_directory())
assert seg_path.exists(), f"Result file {seg_path} was not created"
def test_split_by_subject_ids_invalid(splits: List[List[str]]) -> None:
df1 = pd.read_csv(full_ml_test_data_path(DATASET_CSV_FILE_NAME), dtype=str)
with pytest.raises(ValueError):
DatasetSplits.from_subject_ids(df1,
train_ids=splits[0],
val_ids=splits[1],
test_ids=splits[2])
def test_download_or_get_local_file_2(
test_output_dirs: OutputFolderForTests) -> None:
config = PlotCrossValidationConfig(
run_recovery_id=None,
model_category=ModelCategory.Classification,
epoch=None,
should_validate=False)
download_to_folder = test_output_dirs.root_dir / CROSSVAL_RESULTS_FOLDER
config.outputs_directory = download_to_folder
local_results = full_ml_test_data_path(
"plot_cross_validation") / "HD_cfff5ceb-a227-41d6-a23c-0ebbc33b6301"
config.local_run_results = str(local_results)
# A file that sits in the root folder of the local_results should be downloaded into the
# root of the download_to folder
file1 = "dummy.txt"
file_in_folder = config.download_or_get_local_file(None, file1,
download_to_folder)
assert file_in_folder is not None
assert file_in_folder == download_to_folder / file1
# Copying a file in a sub-folder of the local_results: The full path to the file should be
# preserved and created in the download_to folder.
file2 = Path("0") / "Val" / "metrics.csv"
file_in_folder = config.download_or_get_local_file(None, file2,
download_to_folder)
assert file_in_folder is not None
assert file_in_folder == download_to_folder / file2
def test_plot_image_and_multiple_contours(
test_output_dirs: OutputFolderForTests) -> None:
"""
Test plotting of an image with two overlaid contours.
"""
size = (3, 3)
image = np.zeros(size)
image[0, 0] = -1
image[2, 2] = 1
labels1 = np.zeros(size)
labels1[1, 1] = 1
labels2 = np.zeros(size)
labels2[0, 0] = 1
file_name = "image_and_multiple_contours.png"
plot_file = test_output_dirs.root_dir / file_name
args1 = {'colors': 'r', 'linestyles': 'dashed'}
args2 = {'colors': 'b'}
plotting.plot_image_and_label_contour(image, [labels1, labels2],
contour_arguments=[args1, args2],
plot_file_name=plot_file)
assert plot_file.exists()
expected = full_ml_test_data_path(file_name)
# To update the stored results, uncomment this line:
# expected.write_bytes(plot_file.read_bytes())
assert file_as_bytes(plot_file) == file_as_bytes(expected)
def test_submit_for_inference(test_output_dirs: OutputFolderForTests) -> None:
"""
Execute the submit_for_inference script on the model that was recently trained. This starts an AzureML job,
and downloads the segmentation. Then check if the segmentation was actually produced.
:return:
"""
model = get_most_recent_model(
fallback_run_id_for_local_execution=FALLBACK_SINGLE_RUN)
image_file = fixed_paths_for_tests.full_ml_test_data_path(
) / "train_and_test_data" / "id1_channel1.nii.gz"
assert image_file.exists(), f"Image file not found: {image_file}"
settings_file = fixed_paths.SETTINGS_YAML_FILE
assert settings_file.exists(), f"Settings file not found: {settings_file}"
# Read the name of the branch from environment, so that the inference experiment is also listed alongside
# all other AzureML runs that belong to the current PR.
build_branch = os.environ.get("BUILD_BRANCH", None)
experiment_name = to_azure_friendly_string(
build_branch) if build_branch else "model_inference"
args = [
"--image_file",
str(image_file), "--model_id", model.id, "--settings",
str(settings_file), "--download_folder",
str(test_output_dirs.root_dir), "--cluster", "training-nc12",
"--experiment", experiment_name
]
seg_path = test_output_dirs.root_dir / DEFAULT_RESULT_IMAGE_NAME
assert not seg_path.exists(
), f"Result file {seg_path} should not yet exist"
submit_for_inference.main(
args, project_root=fixed_paths.repository_root_directory())
assert seg_path.exists(), f"Result file {seg_path} was not created"
def _load_and_scale_image(name: str) -> ImageWithHeader:
image_with_header = load_nifti_image(full_ml_test_data_path(name))
return ImageWithHeader(image=LinearTransform.transform(
data=image_with_header.image,
input_range=(0, 255),
output_range=(0, 1)),
header=image_with_header.header)
def test_split_by_institution_invalid(splits: List[float]) -> None:
df1 = pd.read_csv(full_ml_test_data_path(DATASET_CSV_FILE_NAME))
with pytest.raises(ValueError):
DatasetSplits.from_institutions(df1,
splits[0],
splits[1],
splits[2],
shuffle=False)
def test_register_and_score_model(test_output_dirs: OutputFolderForTests) -> None:
"""
End-to-end test which ensures the scoring pipeline is functioning as expected when used on a recently created
model. This test is run after training an ensemble run in AzureML. It starts "submit_for_inference" via
Popen. The inference run here is on a 2-channel model, whereas test_submit_for_inference works with a 1-channel
model.
"""
azureml_model = get_most_recent_model(fallback_run_id_for_local_execution=FALLBACK_ENSEMBLE_RUN)
assert azureml_model is not None
assert PYTHON_ENVIRONMENT_NAME in azureml_model.tags, "Environment name not present in model properties"
# download the registered model and test that we can run the score pipeline on it
model_root = Path(azureml_model.download(str(test_output_dirs.root_dir)))
# The model needs to contain score.py at the root, the (merged) environment definition,
# and the inference config.
expected_files = [
*fixed_paths.SCRIPTS_AT_ROOT,
fixed_paths.ENVIRONMENT_YAML_FILE_NAME,
fixed_paths.MODEL_INFERENCE_JSON_FILE_NAME,
"InnerEye/ML/runner.py",
]
for expected_file in expected_files:
assert (model_root / expected_file).is_file(), f"File {expected_file} missing"
checkpoint_folder = model_root / CHECKPOINT_FOLDER
assert checkpoint_folder.is_dir()
checkpoints = list(checkpoint_folder.rglob("*"))
assert len(checkpoints) >= 1, "There must be at least 1 checkpoint"
# create a dummy datastore to store the image data
test_datastore = test_output_dirs.root_dir / "test_datastore"
# move test data into the data folder to simulate an actual run
train_and_test_data_dir = full_ml_test_data_path("train_and_test_data")
img_files = ["id1_channel1.nii.gz", "id1_channel2.nii.gz"]
data_root = test_datastore / fixed_paths.DEFAULT_DATA_FOLDER
data_root.mkdir(parents=True)
for f in img_files:
shutil.copy(str(train_and_test_data_dir / f), str(data_root))
# run score pipeline as a separate process
python_executable = sys.executable
[return_code1, stdout1] = spawn_and_monitor_subprocess(process=python_executable,
args=["--version"])
assert return_code1 == 0
print(f"Executing Python version {stdout1[0]}")
return_code, stdout2 = spawn_and_monitor_subprocess(process=python_executable, args=[
str(model_root / fixed_paths.SCORE_SCRIPT),
f"--data_folder={str(data_root)}",
f"--image_files={img_files[0]},{img_files[1]}",
"--use_gpu=False"])
# check that the process completed as expected
assert return_code == 0, f"Subprocess failed with return code {return_code}. Stdout: {os.linesep.join(stdout2)}"
expected_segmentation_path = Path(model_root) / DEFAULT_RESULT_IMAGE_NAME
assert expected_segmentation_path.exists(), f"Result file not found: {expected_segmentation_path}"
# sanity check the resulting segmentation
expected_shape = get_nifti_shape(train_and_test_data_dir / img_files[0])
image_header = get_unit_image_header()
assert_nifti_content(str(expected_segmentation_path), expected_shape, image_header, [3], np.ubyte)
def test_load_and_stack(file_path_str: str, expected_shape: Tuple) -> None:
file_path = Path(file_path_str)
files = [full_ml_test_data_path() / f for f in [file_path, file_path]]
stacked = load_images_and_stack(files, load_segmentation=False)
assert torch.is_tensor(stacked.segmentations)
assert stacked.segmentations is not None
assert stacked.segmentations.shape == (0, )
assert torch.is_tensor(stacked.images)
assert stacked.images.shape == (2, ) + expected_shape
def test_load_items_seq_from_dataset() -> None:
"""
Test loading a sequence dataset with numerical, categorical features and images.
"""
dummy_dataset = full_ml_test_data_path(
) / "sequence_data_for_classification" / "dataset.csv"
df = pd.read_csv(dummy_dataset, sep=",", dtype=str)
items: List[SequenceDataSource] = DataSourceReader[SequenceDataSource](
data_frame=df,
image_channels=None,
image_file_column="IMG",
label_channels=None,
label_value_column="Label",
numerical_columns=["NUM1", "NUM2", "NUM3", "NUM4"],
sequence_column="Position").load_data_sources()
assert len(items) == 3 * 9 # 3 subjects, 9 visits each, no missing
assert items[0].metadata.id == "2137.00005"
assert items[0].metadata.sequence_position == 0
assert items[0].metadata.props["CAT2"] == "category_A"
# One of the labels is missing, missing labels should be encoded as NaN
assert math.isnan(items[0].label[0])
assert items[0].channel_files == ["img_1"]
assert str(items[0].numerical_non_image_features.tolist()) == str(
[362.0, np.nan, np.nan, 71.0])
assert items[8].metadata.id == "2137.00005"
assert items[8].metadata.sequence_position == 8
assert items[8].label.tolist() == [0.0]
assert items[8].channel_files == ['']
assert str(items[8].numerical_non_image_features.tolist()) == str(
[350.0, np.nan, np.nan, 8.0])
assert items[16].metadata.id == "2627.00001"
assert items[16].label.tolist() == [0.0]
assert items[16].channel_files == ["img_2"]
assert_tensors_equal(items[16].numerical_non_image_features,
[217.0, 0.0, 0.01, 153.0])
assert items[26].metadata.id == "3250.00005"
assert items[26].metadata.sequence_position == 8
assert_tensors_equal(items[26].label, [0.0])
assert items[26].channel_files == ["img_11"]
assert_tensors_equal(items[26].numerical_non_image_features,
[238.0, 0.0, 0.02, 84.0])
grouped = group_samples_into_sequences(
filter_valid_classification_data_sources_items(
items, file_to_path_mapping=None,
max_sequence_position_value=None))
# There are 3 patients total, but one of them has missing measurements for all visits
assert len(grouped) == 2
assert grouped[0].id == "2627.00001"
assert grouped[1].id == "3250.00005"
# 2627.00001 has full information for weeks 0, 4, and 8
assert len(grouped[0].items) == 3
assert grouped[0].items[0].metadata["VISIT"] == "V1"
assert grouped[0].items[2].metadata["VISIT"] == "VST 3"
assert len(grouped[1].items) == 9
assert items[16].metadata.sequence_position == 7
def _get_metrics_df(mode: ModelExecutionMode) -> pd.DataFrame:
metrics_df = pd.read_csv(
full_ml_test_data_path("{}_agg_splits.csv".format(mode.value)))
# noinspection PyUnresolvedReferences
metrics_df.split = [
DEFAULT_ENSEMBLE_RUN_RECOVERY_ID + "_" + index
for index in metrics_df.split.astype(str)
]
return metrics_df.sort_values(list(metrics_df.columns),
ascending=True).reset_index(drop=True)
def compare_files(actual: List[Path], expected: List[str]) -> None:
assert len(actual) == len(expected)
for (f, e) in zip(actual, expected):
assert f.exists()
full_expected = full_ml_test_data_path(e)
assert full_expected.exists()
assert str(f).endswith(e)
# To update the stored results, uncomment this line:
# full_expected.write_bytes(f.read_bytes())
assert file_as_bytes(f) == file_as_bytes(full_expected)
def test_try_create_mean_teacher_model_and_load_from_checkpoint(config: ModelConfigBase, checkpoint_path: str) -> None:
config.mean_teacher_alpha = 0.999
# no checkpoint path provided
model_and_info = ModelAndInfo(config,
model_execution_mode=ModelExecutionMode.TEST,
checkpoint_path=None)
with pytest.raises(ValueError):
model_and_info.mean_teacher_model
model_loaded = model_and_info.try_create_mean_teacher_model_and_load_from_checkpoint()
assert model_loaded
if isinstance(config, SegmentationModelBase):
assert isinstance(model_and_info.mean_teacher_model, BaseModel)
else:
assert isinstance(model_and_info.mean_teacher_model, DeviceAwareModule)
# Invalid checkpoint path provided
model_and_info = ModelAndInfo(config,
model_execution_mode=ModelExecutionMode.TEST,
checkpoint_path=full_ml_test_data_path("non_exist.pth.tar"))
model_loaded = model_and_info.try_create_mean_teacher_model_and_load_from_checkpoint()
assert not model_loaded
# Current code assumes that even if this function returns False, the model itself was created, only the checkpoint
# loading failed.
if isinstance(config, SegmentationModelBase):
assert isinstance(model_and_info.mean_teacher_model, BaseModel)
else:
assert isinstance(model_and_info.mean_teacher_model, DeviceAwareModule)
# Valid checkpoint path provided
model_and_info = ModelAndInfo(config,
model_execution_mode=ModelExecutionMode.TEST,
checkpoint_path=full_ml_test_data_path(checkpoint_path))
model_loaded = model_and_info.try_create_mean_teacher_model_and_load_from_checkpoint()
assert model_loaded
if isinstance(config, SegmentationModelBase):
assert isinstance(model_and_info.mean_teacher_model, BaseModel)
else:
assert isinstance(model_and_info.mean_teacher_model, DeviceAwareModule)
assert model_and_info.checkpoint_epoch == 1
def test_metrics_file(test_output_dirs: OutputFolderForTests) -> None:
"""Test if metrics files with Dice scores are written as expected."""
folder = test_output_dirs.make_sub_dir("test_metrics_file")
def new_file(suffix: str) -> Path:
file = folder / suffix
if file.is_file():
file.unlink()
return file
d = MetricsPerPatientWriter()
p1 = "Patient1"
p2 = "Patient2"
p3 = "Patient3"
liver = "liver"
kidney = "kidney"
# Ordering for test data: For "liver", patient 2 has the lowest score, sorting should move them first
# For "kidney", patient 1 has the lowest score and should be first.
d.add(p1, liver, 1.0, 1.0, 0.5)
d.add(p1, liver, 0.4, 1.0, 0.4)
d.add(p2, liver, 0.8, 1.0, 0.3)
d.add(p2, kidney, 0.7, 1.0, 0.2)
d.add(p3, kidney, 0.4, 1.0, 0.1)
metrics_file = new_file("metrics_file.csv")
d.to_csv(Path(metrics_file))
# Sorting should be first by structure name alphabetically, then Dice with lowest scores first.
assert_file_contains_string(
metrics_file,
"Patient,Structure,Dice,HausdorffDistance_mm,MeanDistance_mm\n"
"Patient3,kidney,0.400,1.000,0.100\n"
"Patient2,kidney,0.700,1.000,0.200\n"
"Patient1,liver,0.400,1.000,0.400\n"
"Patient2,liver,0.800,1.000,0.300\n"
"Patient1,liver,1.000,1.000,0.500\n")
aggregates_file = new_file(METRICS_AGGREGATES_FILE)
d.save_aggregates_to_csv(Path(aggregates_file))
# Sorting should be first by structure name alphabetically, then Dice with lowest scores first.
assert_text_files_match(Path(aggregates_file),
full_ml_test_data_path() / METRICS_AGGREGATES_FILE)
boxplot_per_structure(d.to_data_frame(),
column_name=MetricsFileColumns.DiceNumeric.value,
title="Dice score")
boxplot1 = new_file("boxplot_2class.png")
resize_and_save(5, 4, boxplot1)
plt.clf()
d.add(p1, "lung", 0.5, 2.0, 1.0)
d.add(p1, "foo", 0.9, 2.0, 1.0)
d.add(p1, "bar", 0.9, 2.0, 1.0)
d.add(p1, "baz", 0.9, 2.0, 1.0)
boxplot_per_structure(d.to_data_frame(),
column_name=MetricsFileColumns.DiceNumeric.value,
title="Dice score")
boxplot2 = new_file("boxplot_6class.png")
resize_and_save(5, 4, boxplot2)
def test_save_outliers(test_config: PlotCrossValidationConfig,
test_output_dirs: OutputFolderForTests) -> None:
"""Test to make sure the outlier file for a split is as expected"""
test_config.outputs_directory = test_output_dirs.root_dir
test_config.outlier_range = 0
assert test_config.run_recovery_id
dataset_split_metrics = {x: _get_metrics_df(test_config.run_recovery_id, x) for x in [ModelExecutionMode.VAL]}
save_outliers(test_config, dataset_split_metrics, test_config.outputs_directory)
f = f"{ModelExecutionMode.VAL.value}_outliers.txt"
assert_text_files_match(full_file=test_config.outputs_directory / f,
expected_file=full_ml_test_data_path(f))
def test_nii_load_zyx(test_output_dirs: OutputFolderForTests) -> None:
expected_shape = (44, 167, 167)
file_path = full_ml_test_data_path("patch_sampling/scan_small.nii.gz")
image: sitk.Image = sitk.ReadImage(str(file_path))
assert image.GetSize() == reverse_tuple_float3(expected_shape)
img = sitk.GetArrayFromImage(image)
assert img.shape == expected_shape
image_header = io_util.load_nifti_image(file_path)
assert image_header.image.shape == expected_shape
assert image_header.header.spacing is not None
np.testing.assert_allclose(image_header.header.spacing, (3.0, 1.0, 1.0), rtol=0.1)
def test_load_and_stack_with_segmentation() -> None:
expected_shape = (4, 5, 7)
file_path = full_ml_test_data_path() / "hdf5_data/patient_hdf5s/4be9beed-5861-fdd2-72c2-8dd89aadc1ef.h5"
files = [file_path, file_path]
stacked = load_images_and_stack(files, load_segmentation=True)
assert stacked.segmentations is not None
assert torch.is_tensor(stacked.segmentations)
assert stacked.segmentations.dtype == torch.uint8
assert stacked.segmentations.shape == (2,) + expected_shape
assert torch.is_tensor(stacked.images)
assert stacked.images.dtype == torch.float16
assert stacked.images.shape == (2,) + expected_shape
def test_model_inference_train_and_test(
test_output_dirs: OutputFolderForTests, perform_cross_validation: bool,
perform_training_set_inference: bool) -> None:
config = DummyModel()
config.number_of_cross_validation_splits = 2 if perform_cross_validation else 0
config.perform_training_set_inference = perform_training_set_inference
# Plotting crashes with random TCL errors on Windows, disable that for Windows PR builds.
config.is_plotting_enabled = common_util.is_linux()
config.set_output_to(test_output_dirs.root_dir)
config.local_dataset = full_ml_test_data_path()
# To make it seem like there was a training run before this, copy checkpoints into the checkpoints folder.
stored_checkpoints = full_ml_test_data_path("checkpoints")
shutil.copytree(str(stored_checkpoints), str(config.checkpoint_folder))
checkpoint_handler = get_default_checkpoint_handler(
model_config=config, project_root=test_output_dirs.root_dir)
checkpoint_handler.additional_training_done()
result, _, _ = MLRunner(config).model_inference_train_and_test(
checkpoint_handler=checkpoint_handler)
if result is None:
raise ValueError("Error result cannot be None")
assert isinstance(result, InferenceMetricsForSegmentation)
for key, _ in result.epochs.items():
epoch_folder_name = common_util.epoch_folder_name(key)
for folder in [
ModelExecutionMode.TRAIN.value, ModelExecutionMode.VAL.value,
ModelExecutionMode.TEST.value
]:
results_folder = config.outputs_folder / epoch_folder_name / folder
folder_exists = results_folder.is_dir()
if folder in [
ModelExecutionMode.TRAIN.value,
ModelExecutionMode.VAL.value
]:
if perform_training_set_inference:
assert folder_exists
else:
assert folder_exists
