def test_run_ml_with_classification_model(
test_output_dirs: OutputFolderForTests,
number_of_offline_cross_validation_splits: int,
model_name: str) -> None:
"""
Test training and testing of classification models, when it is started together via run_ml.
"""
logging_to_stdout()
azure_config = get_default_azure_config()
azure_config.train = True
config: ScalarModelBase = ModelConfigLoader[ScalarModelBase]() \
.create_model_config_from_name(model_name)
config.number_of_cross_validation_splits = number_of_offline_cross_validation_splits
config.set_output_to(test_output_dirs.root_dir)
# Trying to run DDP from the test suite hangs, hence restrict to single GPU.
config.max_num_gpus = 1
MLRunner(config, azure_config).run()
_check_offline_cross_validation_output_files(config)
if config.perform_cross_validation:
# Test that the result files can be correctly picked up by the cross validation routine.
# For that, we point the downloader to the local results folder. The core download method
# recognizes run_recovery_id == None as the signal to read from the local_run_results folder.
config_and_files = get_config_and_results_for_offline_runs(config)
result_files = config_and_files.files
# One file for VAL and one for TRAIN for each child run
assert len(result_files
) == config.get_total_number_of_cross_validation_runs() * 2
for file in result_files:
assert file.execution_mode != ModelExecutionMode.TEST
assert file.dataset_csv_file is not None
assert file.dataset_csv_file.exists()
assert file.metrics_file is not None
assert file.metrics_file.exists()
def spawn_offline_cross_val_classification_child_runs(self) -> None:
"""
Trains and Tests k models based on their respective data splits sequentially.
Stores the results on the Validation set to the outputs directory of the parent run.
"""
_config = self.model_config
assert isinstance(_config, ScalarModelBase)
parent_run_file_system = _config.file_system_config
def _spawn_run(cross_val_split_index: int) -> None:
split_model_config = copy.deepcopy(_config)
assert isinstance(split_model_config, ScalarModelBase)
split_model_config.cross_validation_split_index = cross_val_split_index
_local_split_folder_name = str(cross_val_split_index)
split_model_config.file_system_config = parent_run_file_system.add_subfolder(_local_split_folder_name)
logging.info(f"Running model train and test on cross validation split: {cross_val_split_index}")
split_ml_runner = MLRunner(model_config=split_model_config,
azure_config=self.azure_config,
project_root=self.project_root,
post_cross_validation_hook=self.post_cross_validation_hook,
model_deployment_hook=self.model_deployment_hook)
split_ml_runner.run()
for i in range(_config.number_of_cross_validation_splits):
_spawn_run(i)
config_and_files = get_config_and_results_for_offline_runs(self.model_config)
plot_cross_validation_from_files(config_and_files, Path(config_and_files.config.outputs_directory))
def spawn_offline_cross_val_classification_child_runs(self) -> None:
"""
Trains and Tests k models based on their respective data splits sequentially.
Stores the results on the Validation set to the outputs directory of the parent run.
"""
_config = self.model_config
assert isinstance(_config, ScalarModelBase)
parent_run_file_system = _config.file_system_config
def _spawn_run(cross_val_split_index: int,
cross_val_sub_fold_split_index: int) -> None:
split_model_config = copy.deepcopy(_config)
assert isinstance(split_model_config, ScalarModelBase)
split_model_config.cross_validation_split_index = cross_val_split_index
split_model_config.cross_validation_sub_fold_split_index = cross_val_sub_fold_split_index
if cross_val_sub_fold_split_index == DEFAULT_CROSS_VALIDATION_SPLIT_INDEX:
_local_split_folder_name = str(cross_val_split_index)
else:
_local_split_folder_name = \
str((cross_val_split_index * split_model_config.number_of_cross_validation_splits_per_fold)
+ cross_val_sub_fold_split_index)
split_model_config.file_system_config = parent_run_file_system.add_subfolder(
_local_split_folder_name)
logging.info(
f"Running model train and test on cross validation split: {x}")
split_ml_runner = MLRunner(split_model_config, self.azure_config,
self.project_root,
self.model_deployment_hook,
self.innereye_submodule_name)
split_ml_runner.run()
cv_fold_indices = [
list(range(_config.number_of_cross_validation_splits_per_fold))
if _config.perform_sub_fold_cross_validation else
[DEFAULT_CROSS_VALIDATION_SPLIT_INDEX]
]
cv_fold_indices *= _config.number_of_cross_validation_splits
for i, x in enumerate(cv_fold_indices):
for y in x:
_spawn_run(i, int(y))
config_and_files = get_config_and_results_for_offline_runs(
self.model_config)
plot_cross_validation_from_files(
config_and_files, Path(config_and_files.config.outputs_directory))
def test_run_ml_with_classification_model(
test_output_dirs: TestOutputDirectories,
number_of_offline_cross_validation_splits: int,
number_of_cross_validation_splits_per_fold: int,
model_name: str) -> None:
"""
Test training and testing of classification models, when it is started together via run_ml.
"""
logging_to_stdout()
azure_config = get_default_azure_config()
azure_config.train = True
train_config: ScalarModelBase = ModelConfigLoader[ScalarModelBase]() \
.create_model_config_from_name(model_name)
train_config.number_of_cross_validation_splits = number_of_offline_cross_validation_splits
train_config.number_of_cross_validation_splits_per_fold = number_of_cross_validation_splits_per_fold
train_config.set_output_to(test_output_dirs.root_dir)
if train_config.perform_sub_fold_cross_validation:
train_config.local_dataset = full_ml_test_data_path(
"classification_data_sub_fold_cv")
MLRunner(train_config, azure_config).run()
_check_offline_cross_validation_output_files(train_config)
if train_config.is_regression_model:
assert (train_config.outputs_folder / "0" /
"error_plot_4.png").is_file()
if train_config.perform_cross_validation:
# Test that the result files can be correctly picked up by the cross validation routine.
# For that, we point the downloader to the local results folder. The core download method
# recognizes run_recovery_id == None as the signal to read from the local_run_results folder.
config_and_files = get_config_and_results_for_offline_runs(
train_config)
result_files = config_and_files.files
# One file for VAL and one for TRAIN for each child run
assert len(
result_files
) == train_config.get_total_number_of_cross_validation_runs() * 2
for file in result_files:
assert file.execution_mode != ModelExecutionMode.TEST
assert file.dataset_csv_file is not None
assert file.dataset_csv_file.exists()
assert file.metrics_file is not None
assert file.metrics_file.exists()
def test_model_test(test_output_dirs: OutputFolderForTests) -> None:
train_and_test_data_dir = full_ml_test_data_path("train_and_test_data")
config = DummyModel()
config.set_output_to(test_output_dirs.root_dir)
epoch = 1
config.num_epochs = epoch
assert config.get_test_epochs() == [epoch]
placeholder_dataset_id = "place_holder_dataset_id"
config.azure_dataset_id = placeholder_dataset_id
transform = config.get_full_image_sample_transforms().test
df = pd.read_csv(full_ml_test_data_path(DATASET_CSV_FILE_NAME))
df = df[df.subject.isin([1, 2])]
# noinspection PyTypeHints
config._datasets_for_inference = \
{ModelExecutionMode.TEST: FullImageDataset(config, df, full_image_sample_transforms=transform)} # type: ignore
execution_mode = ModelExecutionMode.TEST
checkpoint_handler = get_default_checkpoint_handler(model_config=config,
project_root=test_output_dirs.root_dir)
# Mimic the behaviour that checkpoints are downloaded from blob storage into the checkpoints folder.
stored_checkpoints = full_ml_test_data_path("checkpoints")
shutil.copytree(str(stored_checkpoints), str(config.checkpoint_folder))
checkpoint_handler.additional_training_done()
inference_results = model_testing.segmentation_model_test(config,
data_split=execution_mode,
checkpoint_handler=checkpoint_handler)
epoch_dir = config.outputs_folder / get_epoch_results_path(epoch, execution_mode)
assert inference_results.epochs[epoch] == pytest.approx(0.66606902, abs=1e-6)
assert config.outputs_folder.is_dir()
assert epoch_dir.is_dir()
patient1 = io_util.load_nifti_image(train_and_test_data_dir / "id1_channel1.nii.gz")
patient2 = io_util.load_nifti_image(train_and_test_data_dir / "id2_channel1.nii.gz")
assert_file_contains_string(epoch_dir / DATASET_ID_FILE, placeholder_dataset_id)
assert_file_contains_string(epoch_dir / GROUND_TRUTH_IDS_FILE, "region")
assert_text_files_match(epoch_dir / model_testing.METRICS_FILE_NAME,
train_and_test_data_dir / model_testing.METRICS_FILE_NAME)
assert_text_files_match(epoch_dir / model_testing.METRICS_AGGREGATES_FILE,
train_and_test_data_dir / model_testing.METRICS_AGGREGATES_FILE)
# Plotting results vary between platforms. Can only check if the file is generated, but not its contents.
assert (epoch_dir / model_testing.BOXPLOT_FILE).exists()
assert_nifti_content(epoch_dir / "001" / "posterior_region.nii.gz", get_image_shape(patient1),
patient1.header,
[136], np.ubyte)
assert_nifti_content(epoch_dir / "002" / "posterior_region.nii.gz", get_image_shape(patient2),
patient2.header,
[136], np.ubyte)
assert_nifti_content(epoch_dir / "001" / DEFAULT_RESULT_IMAGE_NAME, get_image_shape(patient1),
patient1.header,
[1], np.ubyte)
assert_nifti_content(epoch_dir / "002" / DEFAULT_RESULT_IMAGE_NAME, get_image_shape(patient2),
patient2.header,
[1], np.ubyte)
assert_nifti_content(epoch_dir / "001" / "posterior_background.nii.gz", get_image_shape(patient1),
patient1.header,
[118], np.ubyte)
assert_nifti_content(epoch_dir / "002" / "posterior_background.nii.gz", get_image_shape(patient2),
patient2.header,
[118], np.ubyte)
thumbnails_folder = epoch_dir / model_testing.THUMBNAILS_FOLDER
assert thumbnails_folder.is_dir()
png_files = list(thumbnails_folder.glob("*.png"))
overlays = [f for f in png_files if "_region_slice_" in str(f)]
assert len(overlays) == len(df.subject.unique()), "There should be one overlay/contour file per subject"
# Writing dataset.csv normally happens at the beginning of training,
# but this test reads off a saved checkpoint file.
# Dataset.csv must be present for plot_cross_validation.
config.write_dataset_files()
# Test if the metrics files can be picked up correctly by the cross validation code
config_and_files = get_config_and_results_for_offline_runs(config)
result_files = config_and_files.files
assert len(result_files) == 1
for file in result_files:
assert file.execution_mode == execution_mode
assert file.dataset_csv_file is not None
assert file.dataset_csv_file.exists()
assert file.metrics_file is not None
assert file.metrics_file.exists()
def test_model_test(test_output_dirs: OutputFolderForTests,
use_partial_ground_truth: bool,
allow_partial_ground_truth: bool) -> None:
"""
Check the CSVs (and image files) output by InnerEye.ML.model_testing.segmentation_model_test
:param test_output_dirs: The fixture in conftest.py
:param use_partial_ground_truth: Whether to remove some ground truth labels from some test users
:param allow_partial_ground_truth: What to set the allow_incomplete_labels flag to
"""
train_and_test_data_dir = full_ml_test_data_path("train_and_test_data")
seed_everything(42)
config = DummyModel()
config.allow_incomplete_labels = allow_partial_ground_truth
config.set_output_to(test_output_dirs.root_dir)
placeholder_dataset_id = "place_holder_dataset_id"
config.azure_dataset_id = placeholder_dataset_id
transform = config.get_full_image_sample_transforms().test
df = pd.read_csv(full_ml_test_data_path(DATASET_CSV_FILE_NAME))
if use_partial_ground_truth:
config.check_exclusive = False
config.ground_truth_ids = ["region", "region_1"]
# As in Tests.ML.pipelines.test.inference.test_evaluate_model_predictions patients 3, 4,
# and 5 are in the test dataset with:
# Patient 3 has one missing ground truth channel: "region"
df = df[df["subject"].ne(3) | df["channel"].ne("region")]
# Patient 4 has all missing ground truth channels: "region", "region_1"
df = df[df["subject"].ne(4) | df["channel"].ne("region")]
df = df[df["subject"].ne(4) | df["channel"].ne("region_1")]
# Patient 5 has no missing ground truth channels.
config.dataset_data_frame = df
df = df[df.subject.isin([3, 4, 5])]
config.train_subject_ids = ['1', '2']
config.test_subject_ids = ['3', '4', '5']
config.val_subject_ids = ['6', '7']
else:
df = df[df.subject.isin([1, 2])]
if use_partial_ground_truth and not allow_partial_ground_truth:
with pytest.raises(ValueError) as value_error:
# noinspection PyTypeHints
config._datasets_for_inference = {
ModelExecutionMode.TEST:
FullImageDataset(config,
df,
full_image_sample_transforms=transform)
} # type: ignore
assert "Patient 3 does not have channel 'region'" in str(
value_error.value)
return
else:
# noinspection PyTypeHints
config._datasets_for_inference = {
ModelExecutionMode.TEST:
FullImageDataset(config,
df,
full_image_sample_transforms=transform)
} # type: ignore
execution_mode = ModelExecutionMode.TEST
checkpoint_handler = get_default_checkpoint_handler(
model_config=config, project_root=test_output_dirs.root_dir)
# Mimic the behaviour that checkpoints are downloaded from blob storage into the checkpoints folder.
create_model_and_store_checkpoint(
config,
config.checkpoint_folder / LAST_CHECKPOINT_FILE_NAME_WITH_SUFFIX)
checkpoint_handler.additional_training_done()
inference_results = model_testing.segmentation_model_test(
config,
execution_mode=execution_mode,
checkpoint_paths=checkpoint_handler.get_checkpoints_to_test())
epoch_dir = config.outputs_folder / get_best_epoch_results_path(
execution_mode)
total_num_patients_column_name = f"total_{MetricsFileColumns.Patient.value}".lower(
)
if not total_num_patients_column_name.endswith("s"):
total_num_patients_column_name += "s"
if use_partial_ground_truth:
num_subjects = len(pd.unique(df["subject"]))
if allow_partial_ground_truth:
assert csv_column_contains_value(
csv_file_path=epoch_dir / METRICS_AGGREGATES_FILE,
column_name=total_num_patients_column_name,
value=num_subjects,
contains_only_value=True)
assert csv_column_contains_value(
csv_file_path=epoch_dir / SUBJECT_METRICS_FILE_NAME,
column_name=MetricsFileColumns.Dice.value,
value='',
contains_only_value=False)
else:
aggregates_df = pd.read_csv(epoch_dir / METRICS_AGGREGATES_FILE)
assert total_num_patients_column_name not in aggregates_df.columns # Only added if using partial ground truth
assert not csv_column_contains_value(
csv_file_path=epoch_dir / SUBJECT_METRICS_FILE_NAME,
column_name=MetricsFileColumns.Dice.value,
value='',
contains_only_value=False)
assert inference_results.metrics == pytest.approx(0.66606902, abs=1e-6)
assert config.outputs_folder.is_dir()
assert epoch_dir.is_dir()
patient1 = io_util.load_nifti_image(train_and_test_data_dir /
"id1_channel1.nii.gz")
patient2 = io_util.load_nifti_image(train_and_test_data_dir /
"id2_channel1.nii.gz")
assert_file_contains_string(epoch_dir / DATASET_ID_FILE,
placeholder_dataset_id)
assert_file_contains_string(epoch_dir / GROUND_TRUTH_IDS_FILE,
"region")
assert_text_files_match(
epoch_dir / model_testing.SUBJECT_METRICS_FILE_NAME,
train_and_test_data_dir / model_testing.SUBJECT_METRICS_FILE_NAME)
assert_text_files_match(
epoch_dir / model_testing.METRICS_AGGREGATES_FILE,
train_and_test_data_dir / model_testing.METRICS_AGGREGATES_FILE)
# Plotting results vary between platforms. Can only check if the file is generated, but not its contents.
assert (epoch_dir / model_testing.BOXPLOT_FILE).exists()
assert_nifti_content(epoch_dir / "001" / "posterior_region.nii.gz",
get_image_shape(patient1), patient1.header, [137],
np.ubyte)
assert_nifti_content(epoch_dir / "002" / "posterior_region.nii.gz",
get_image_shape(patient2), patient2.header, [137],
np.ubyte)
assert_nifti_content(epoch_dir / "001" / DEFAULT_RESULT_IMAGE_NAME,
get_image_shape(patient1), patient1.header, [1],
np.ubyte)
assert_nifti_content(epoch_dir / "002" / DEFAULT_RESULT_IMAGE_NAME,
get_image_shape(patient2), patient2.header, [1],
np.ubyte)
assert_nifti_content(epoch_dir / "001" / "posterior_background.nii.gz",
get_image_shape(patient1), patient1.header, [117],
np.ubyte)
assert_nifti_content(epoch_dir / "002" / "posterior_background.nii.gz",
get_image_shape(patient2), patient2.header, [117],
np.ubyte)
thumbnails_folder = epoch_dir / model_testing.THUMBNAILS_FOLDER
assert thumbnails_folder.is_dir()
png_files = list(thumbnails_folder.glob("*.png"))
overlays = [f for f in png_files if "_region_slice_" in str(f)]
assert len(overlays) == len(df.subject.unique(
)), "There should be one overlay/contour file per subject"
# Writing dataset.csv normally happens at the beginning of training,
# but this test reads off a saved checkpoint file.
# Dataset.csv must be present for plot_cross_validation.
config.write_dataset_files()
# Test if the metrics files can be picked up correctly by the cross validation code
config_and_files = get_config_and_results_for_offline_runs(config)
result_files = config_and_files.files
assert len(result_files) == 1
for file in result_files:
assert file.execution_mode == execution_mode
assert file.dataset_csv_file is not None
assert file.dataset_csv_file.exists()
assert file.metrics_file is not None
assert file.metrics_file.exists()
