def test_create_python_env() -> None:
"""
Checks if environment variables in the SourceConfig are correctly passed through to the Python environment.
Environment variables in SourceConfig are only used in the internal InnerEye repo.
:return:
"""
foo = "foo"
bar = "bar"
entry_script = Path("something.py")
conda_file = get_environment_yaml_file()
s = SourceConfig(root_folder=Path(""),
entry_script=entry_script,
conda_dependencies_files=[conda_file],
environment_variables={foo: bar})
env = get_or_create_python_environment(
source_config=s,
azure_config=get_default_azure_config(),
register_environment=False)
assert foo in env.environment_variables
assert env.environment_variables[foo] == bar
# Check that some of the basic packages that we expect to always exist are picked up correctly in the Conda env
def remove_version_number(items: Iterator[str]) -> Set[str]:
return set(c.split("=")[0] for c in items)
assert "pytorch" in remove_version_number(
env.python.conda_dependencies.conda_packages)
assert "pytorch-lightning" in remove_version_number(
env.python.conda_dependencies.pip_packages)
def test_run_ml_with_classification_model(
test_output_dirs: OutputFolderForTests,
number_of_offline_cross_validation_splits: int,
model_name: str) -> None:
"""
Test training and testing of classification models, when it is started together via run_ml.
"""
logging_to_stdout()
azure_config = get_default_azure_config()
azure_config.train = True
config: ScalarModelBase = ModelConfigLoader[ScalarModelBase]() \
.create_model_config_from_name(model_name)
config.number_of_cross_validation_splits = number_of_offline_cross_validation_splits
config.set_output_to(test_output_dirs.root_dir)
# Trying to run DDP from the test suite hangs, hence restrict to single GPU.
config.max_num_gpus = 1
MLRunner(config, azure_config).run()
_check_offline_cross_validation_output_files(config)
if config.perform_cross_validation:
# Test that the result files can be correctly picked up by the cross validation routine.
# For that, we point the downloader to the local results folder. The core download method
# recognizes run_recovery_id == None as the signal to read from the local_run_results folder.
config_and_files = get_config_and_results_for_offline_runs(config)
result_files = config_and_files.files
# One file for VAL and one for TRAIN for each child run
assert len(result_files
) == config.get_total_number_of_cross_validation_runs() * 2
for file in result_files:
assert file.execution_mode != ModelExecutionMode.TEST
assert file.dataset_csv_file is not None
assert file.dataset_csv_file.exists()
assert file.metrics_file is not None
assert file.metrics_file.exists()
def test_run_ml_with_sequence_model(use_combined_model: bool,
imaging_feature_type: ImagingFeatureType,
test_output_dirs: TestOutputDirectories) -> None:
"""
Test training and testing of sequence models, when it is started together via run_ml.
"""
logging_to_stdout()
config = ToySequenceModel(use_combined_model, imaging_feature_type,
should_validate=False, sequence_target_positions=[2, 10])
config.set_output_to(test_output_dirs.root_dir)
config.dataset_data_frame = _get_mock_sequence_dataset()
config.num_epochs = 1
config.max_batch_grad_cam = 1
# make sure we are testing with at least one sequence position that will not exist
# to ensure correct handling of sequences that do not contain all the expected target positions
assert max(config.sequence_target_positions) > config.dataset_data_frame[config.sequence_column].astype(float).max()
# Patch the load_images function that will be called once we access a dataset item
image_and_seg = ImageAndSegmentations[np.ndarray](images=np.random.uniform(0, 1, SCAN_SIZE),
segmentations=np.random.randint(0, 2, SCAN_SIZE))
with mock.patch('InnerEye.ML.utils.io_util.load_image_in_known_formats', return_value=image_and_seg):
azure_config = get_default_azure_config()
azure_config.train = True
MLRunner(config, azure_config).run()
def test_download_pytest_file(test_output_dirs: OutputFolderForTests) -> None:
output_dir = test_output_dirs.root_dir
azure_config = get_default_azure_config()
workspace = azure_config.get_workspace()
def get_run_and_download_pytest(branch: str,
number: int) -> Optional[Path]:
experiment = Experiment(workspace,
name=to_azure_friendly_string(branch))
runs = [run for run in experiment.get_runs() if run.number == number]
if len(runs) != 1:
raise ValueError(
f"Expected to get exactly 1 run in experiment {experiment.name}"
)
return download_pytest_result(runs[0], output_dir)
# PR 49 is a recent successful build that generated a pytest file.
# Run 6 in that experiment was canceled did not yet write the pytest file:
with pytest.raises(ValueError) as ex:
get_run_and_download_pytest("refs/pull/219/merge", 6)
assert "No pytest result file" in str(ex)
downloaded = get_run_and_download_pytest("refs/pull/219/merge", 7)
assert downloaded is not None
assert downloaded.exists()
# Delete the file - it should be cleaned up with the test output directories though.
# If the file remained, it would be uploaded as a test result file to Azure DevOps
downloaded.unlink()
def test_dataset_consumption1() -> None:
"""
Creating datasets, case 1: Azure datasets given, no local folders or mount points
:return:
"""
azure_config = get_default_azure_config()
datasets = create_dataset_configs(azure_config,
all_azure_dataset_ids=["1", "2"],
all_dataset_mountpoints=[],
all_local_datasets=[])
assert len(datasets) == 2
assert datasets[0].name == "1"
assert datasets[1].name == "2"
for i in range(2):
assert datasets[i].local_folder is None
assert datasets[i].target_folder is None
# Two error cases: number of mount points or number of local datasets does not match
with pytest.raises(ValueError) as ex:
create_dataset_configs(azure_config,
all_azure_dataset_ids=["1", "2"],
all_dataset_mountpoints=["mp"],
all_local_datasets=[])
assert "Invalid dataset setup" in str(ex)
with pytest.raises(ValueError) as ex:
create_dataset_configs(azure_config,
all_azure_dataset_ids=["1", "2"],
all_dataset_mountpoints=[],
all_local_datasets=[Path("local")])
assert "Invalid dataset setup" in str(ex)
def test_dataset_consumption5() -> None:
"""
Test error handling for empty dataset IDs.
"""
azure_config = get_default_azure_config()
with pytest.raises(ValueError) as ex:
azure_config.get_or_create_dataset("")
assert "No dataset ID provided" in str(ex)
def test_dataset_consumption2() -> None:
"""
Test that given mount point with empty dataset ID raises an error
"""
azure_config = get_default_azure_config()
with pytest.raises(ValueError) as ex:
create_dataset_consumptions(azure_config, [""], ["foo"])
assert "but a mount point has been provided" in str(ex)
def test_dataset_consumption3() -> None:
"""
Test that a matching number of mount points is created.
"""
azure_config = get_default_azure_config()
assert len(
create_dataset_consumptions(azure_config,
["test-dataset", "test-dataset"], [])) == 2
def get_most_recent_run(fallback_run_id_for_local_execution: str = FALLBACK_SINGLE_RUN) -> Run:
"""
Gets the name of the most recently executed AzureML run, instantiates that Run object and returns it.
:param fallback_run_id_for_local_execution: A hardcoded AzureML run ID that is used when executing this code
on a local box, outside of Azure build agents.
"""
run_recovery_id = get_most_recent_run_id(fallback_run_id_for_local_execution=fallback_run_id_for_local_execution)
return get_default_azure_config().fetch_run(run_recovery_id=run_recovery_id)
def test_dataset_consumption4() -> None:
"""
Test error handling for number of mount points.
"""
azure_config = get_default_azure_config()
with pytest.raises(ValueError) as ex:
create_dataset_consumptions(azure_config,
["test-dataset", "test-dataset"], ["foo"])
assert "must equal the number of Azure dataset IDs" in str(ex)
def runner_config() -> AzureConfig:
"""
Gets an Azure config that masks out the storage account for datasets, to avoid accidental overwriting.
:return:
"""
config = get_default_azure_config()
config.model = ""
config.train = False
return config
def test_get_comparison_data(test_output_dirs: TestOutputDirectories) -> None:
azure_config = get_default_azure_config()
comparison_name = "DefaultName"
comparison_path = DEFAULT_RUN_RECOVERY_ID + "/outputs/epoch_002/Test"
baselines = get_comparison_baselines(Path(test_output_dirs.root_dir),
azure_config,
[(comparison_name, comparison_path)])
assert len(baselines) == 1
assert baselines[0].name == comparison_name
def test_download_dataset_via_blobxfer(test_output_dirs: TestOutputDirectories) -> None:
azure_config = get_default_azure_config()
result_path = run_ml.download_dataset_via_blobxfer(dataset_id="test-dataset",
azure_config=azure_config,
target_folder=Path(test_output_dirs.root_dir))
assert result_path
assert result_path.is_dir()
dataset_csv = Path(result_path) / DATASET_CSV_FILE_NAME
assert dataset_csv.exists()
def test_dataset_consumption4() -> None:
"""
Creating datasets, case 4: no datasets at all
"""
azure_config = get_default_azure_config()
datasets = create_dataset_configs(azure_config,
all_azure_dataset_ids=[],
all_dataset_mountpoints=[],
all_local_datasets=[])
assert len(datasets) == 0
def test_get_comparison_data(test_output_dirs: OutputFolderForTests) -> None:
azure_config = get_default_azure_config()
comparison_name = "DefaultName"
comparison_path = get_most_recent_run_id() + \
f"/{DEFAULT_AML_UPLOAD_DIR}/{BEST_EPOCH_FOLDER_NAME}/{ModelExecutionMode.TEST.value}"
baselines = get_comparison_baselines(test_output_dirs.root_dir,
azure_config,
[(comparison_name, comparison_path)])
assert len(baselines) == 1
assert baselines[0].name == comparison_name
def test_dataset_consumption3() -> None:
"""
Creating datasets, case 3: local datasets only. This should generate no results
"""
azure_config = get_default_azure_config()
datasets = create_dataset_configs(azure_config,
all_azure_dataset_ids=[],
all_dataset_mountpoints=[],
all_local_datasets=[Path("l1"), Path("l2")])
assert len(datasets) == 0
def test_run_ml_with_multi_label_sequence_model(
test_output_dirs: OutputFolderForTests) -> None:
"""
Test training and testing of sequence models that predicts at multiple time points,
when it is started via run_ml.
"""
logging_to_stdout()
config = ToyMultiLabelSequenceModel(should_validate=False)
assert config.get_target_indices() == [1, 2, 3]
expected_prediction_targets = [
f"{SEQUENCE_POSITION_HUE_NAME_PREFIX} {x}" for x in ["01", "02", "03"]
]
_target_indices = config.get_target_indices()
assert _target_indices is not None
assert len(_target_indices) == len(expected_prediction_targets)
metrics_dict = create_metrics_dict_for_scalar_models(config)
assert metrics_dict.get_hue_names(
include_default=False) == expected_prediction_targets
config.set_output_to(test_output_dirs.root_dir)
# Create a fake dataset directory to make config validation pass
config.local_dataset = test_output_dirs.root_dir
config.dataset_data_frame = _get_multi_label_sequence_dataframe()
config.pre_process_dataset_dataframe()
config.num_epochs = 1
config.max_batch_grad_cam = 1
azure_config = get_default_azure_config()
azure_config.train = True
MLRunner(config, azure_config).run()
# The metrics file should have one entry per epoch per subject per prediction target,
# for all the 3 prediction targets.
metrics_file = config.outputs_folder / "Train" / SUBJECT_METRICS_FILE_NAME
assert metrics_file.exists()
metrics = pd.read_csv(metrics_file)
assert LoggingColumns.Patient.value in metrics
assert LoggingColumns.Epoch.value in metrics
assert LoggingColumns.Hue.value in metrics
assert metrics[LoggingColumns.Hue.value].unique().tolist(
) == expected_prediction_targets
group_by_subject = metrics.groupby(
by=[LoggingColumns.Patient.value, LoggingColumns.Epoch.value])
expected_prediction_target_lengths = [3, 2, 3, 3]
for i, x in enumerate(group_by_subject):
assert len(x[1]) == expected_prediction_target_lengths[i]
group_by_subject_and_target = metrics.groupby(by=[
LoggingColumns.Patient.value, LoggingColumns.Epoch.value,
LoggingColumns.Hue.value
])
for _, group in group_by_subject_and_target:
assert len(group) == 1
def test_download_or_get_local_blobs(is_current_run: bool,
test_config: PlotCrossValidationConfig,
test_output_dirs: OutputFolderForTests) -> None:
azure_config = get_default_azure_config()
azure_config.get_workspace()
assert test_config.run_recovery_id is not None
run = Run.get_context() if is_current_run else azure_config.fetch_run(test_config.run_recovery_id)
run_outputs_dir = full_ml_test_data_path() if is_current_run else Path(DEFAULT_AML_UPLOAD_DIR)
test_config.outputs_directory = run_outputs_dir
dst = test_config.download_or_get_local_file(
blob_to_download="dataset.csv",
destination=test_output_dirs.root_dir,
run=run
)
assert dst is not None
assert dst.exists()
def test_image_encoder_with_segmentation(
test_output_dirs: OutputFolderForTests, encode_channels_jointly: bool,
aggregation_type: AggregationType,
imaging_feature_type: ImagingFeatureType) -> None:
"""
Test if the image encoder networks can be trained on segmentations from HDF5.
"""
logging_to_stdout()
set_random_seed(0)
scan_size = (6, 64, 60)
dataset_contents = """subject,channel,path,label
S1,week0,scan1.h5,
S1,week1,scan2.h5,True
S2,week0,scan3.h5,
S2,week1,scan4.h5,False
S3,week0,scan5.h5,
S3,week1,scan6.h5,True
S4,week0,scan7.h5,
S4,week1,scan8.h5,True
"""
config = ImageEncoder(encode_channels_jointly=encode_channels_jointly,
imaging_feature_type=imaging_feature_type,
should_validate=False,
aggregation_type=aggregation_type,
scan_size=scan_size)
# This fails with 16bit precision, saying "torch.nn.functional.binary_cross_entropy and torch.nn.BCELoss are
# unsafe to autocast. Many models use a sigmoid layer right before the binary cross entropy layer. In this case,
# combine the two layers using torch.nn.functional.binary_cross_entropy_with_logits or
# torch.nn.BCEWithLogitsLoss. binary_cross_entropy_with_logits and BCEWithLogits are safe to autocast."
config.use_mixed_precision = False
config.set_output_to(test_output_dirs.root_dir)
config.num_epochs = 1
config.local_dataset = Path()
config.dataset_data_frame = pd.read_csv(StringIO(dataset_contents),
sep=",",
dtype=str)
# Patch the load_images function that will be called once we access a dataset item
image_and_seg = ImageAndSegmentations[np.ndarray](
images=np.zeros(scan_size, dtype=np.float32),
segmentations=np.ones(scan_size, dtype=np.uint8))
with mock.patch('InnerEye.ML.utils.io_util.load_image_in_known_formats',
return_value=image_and_seg):
azure_config = get_default_azure_config()
azure_config.train = True
MLRunner(config, azure_config).run()
def test_download_azureml_dataset(test_output_dirs: OutputFolderForTests) -> None:
dataset_name = "test-dataset"
config = DummyModel()
config.local_dataset = None
config.azure_dataset_id = ""
azure_config = get_default_azure_config()
runner = MLRunner(config, azure_config=azure_config)
# If the model has neither local_dataset or azure_dataset_id, mount_or_download_dataset should fail.
# This mounting call must happen before any other operations on the container, because already the model
# creation may need access to the dataset.
with pytest.raises(ValueError) as ex:
runner.setup()
assert ex.value.args[0] == "The model must contain either local_dataset or azure_dataset_id."
runner.project_root = test_output_dirs.root_dir
# Pointing the model to a dataset folder that does not exist should raise an Exception
fake_folder = runner.project_root / "foo"
runner.container.local_dataset = fake_folder
with pytest.raises(FileNotFoundError):
runner.mount_or_download_dataset(runner.container.azure_dataset_id, runner.container.local_dataset)
# If the local dataset folder exists, mount_or_download_dataset should not do anything.
fake_folder.mkdir()
local_dataset = runner.mount_or_download_dataset(runner.container.azure_dataset_id, runner.container.local_dataset)
assert local_dataset == fake_folder
# Pointing the model to a dataset in Azure should trigger a download
runner.container.local_dataset = None
runner.container.azure_dataset_id = dataset_name
with logging_section("Starting download"):
result_path = runner.mount_or_download_dataset(runner.container.azure_dataset_id,
runner.container.local_dataset)
# Download goes into <project_root> / "datasets" / "test_dataset"
expected_path = runner.project_root / fixed_paths.DATASETS_DIR_NAME / dataset_name
assert result_path == expected_path
assert result_path.is_dir()
dataset_csv = Path(result_path) / DATASET_CSV_FILE_NAME
assert dataset_csv.is_file()
# Check that each individual file in the dataset is present
for folder in [1, *range(10, 20)]:
sub_folder = result_path / str(folder)
sub_folder.is_dir()
for file in ["ct", "esophagus", "heart", "lung_l", "lung_r", "spinalcord"]:
f = (sub_folder / file).with_suffix(".nii.gz")
assert f.is_file()
def test_run_ml_with_segmentation_model(
test_output_dirs: TestOutputDirectories) -> None:
"""
Test training and testing of segmentation models, when it is started together via run_ml.
"""
train_config = DummyModel()
train_config.num_dataload_workers = 0
train_config.restrict_subjects = "1"
# Increasing the test crop size should not have any effect on the results.
# This is for a bug in an earlier version of the code where the wrong execution mode was used to
# compute the expected mask size at training time.
train_config.test_crop_size = (75, 75, 75)
train_config.perform_training_set_inference = False
train_config.perform_validation_and_test_set_inference = True
train_config.set_output_to(test_output_dirs.root_dir)
azure_config = get_default_azure_config()
azure_config.train = True
MLRunner(train_config, azure_config).run()
def test_dataset_consumption2() -> None:
"""
Creating datasets, case 2: Azure datasets, local folders and mount points given
"""
azure_config = get_default_azure_config()
datasets = create_dataset_configs(azure_config,
all_azure_dataset_ids=["1", "2"],
all_dataset_mountpoints=["mp1", "mp2"],
all_local_datasets=[Path("l1"), Path("l2")])
assert len(datasets) == 2
assert datasets[0].name == "1"
assert datasets[1].name == "2"
assert datasets[0].local_folder == Path("l1")
assert datasets[1].local_folder == Path("l2")
if is_linux():
# PosixPath cannot be instantiated on Windows
assert datasets[0].target_folder == PosixPath("mp1")
assert datasets[1].target_folder == PosixPath("mp2")
def test_image_encoder_with_segmentation(
test_output_dirs: OutputFolderForTests, encode_channels_jointly: bool,
aggregation_type: AggregationType,
imaging_feature_type: ImagingFeatureType) -> None:
"""
Test if the image encoder networks can be trained on segmentations from HDF5.
"""
logging_to_stdout()
set_random_seed(0)
scan_size = (6, 64, 60)
dataset_contents = """subject,channel,path,label
S1,week0,scan1.h5,
S1,week1,scan2.h5,True
S2,week0,scan3.h5,
S2,week1,scan4.h5,False
S3,week0,scan5.h5,
S3,week1,scan6.h5,True
S4,week0,scan7.h5,
S4,week1,scan8.h5,True
"""
config = ImageEncoder(encode_channels_jointly=encode_channels_jointly,
imaging_feature_type=imaging_feature_type,
should_validate=False,
aggregation_type=aggregation_type,
scan_size=scan_size)
config.use_mixed_precision = True
config.set_output_to(test_output_dirs.root_dir)
config.num_epochs = 1
config.local_dataset = Path()
config.dataset_data_frame = pd.read_csv(StringIO(dataset_contents),
sep=",",
dtype=str)
# Patch the load_images function that will be called once we access a dataset item
image_and_seg = ImageAndSegmentations[np.ndarray](
images=np.zeros(scan_size, dtype=np.float32),
segmentations=np.ones(scan_size, dtype=np.uint8))
with mock.patch("InnerEye.ML.run_ml.is_offline_run_context",
return_value=True):
with mock.patch(
'InnerEye.ML.utils.io_util.load_image_in_known_formats',
return_value=image_and_seg):
azure_config = get_default_azure_config()
azure_config.train = True
MLRunner(config, azure_config=azure_config).run()
def test_run_ml_with_classification_model(
test_output_dirs: TestOutputDirectories,
number_of_offline_cross_validation_splits: int,
number_of_cross_validation_splits_per_fold: int,
model_name: str) -> None:
"""
Test training and testing of classification models, when it is started together via run_ml.
"""
logging_to_stdout()
azure_config = get_default_azure_config()
azure_config.train = True
train_config: ScalarModelBase = ModelConfigLoader[ScalarModelBase]() \
.create_model_config_from_name(model_name)
train_config.number_of_cross_validation_splits = number_of_offline_cross_validation_splits
train_config.number_of_cross_validation_splits_per_fold = number_of_cross_validation_splits_per_fold
train_config.set_output_to(test_output_dirs.root_dir)
if train_config.perform_sub_fold_cross_validation:
train_config.local_dataset = full_ml_test_data_path(
"classification_data_sub_fold_cv")
MLRunner(train_config, azure_config).run()
_check_offline_cross_validation_output_files(train_config)
if train_config.is_regression_model:
assert (train_config.outputs_folder / "0" /
"error_plot_4.png").is_file()
if train_config.perform_cross_validation:
# Test that the result files can be correctly picked up by the cross validation routine.
# For that, we point the downloader to the local results folder. The core download method
# recognizes run_recovery_id == None as the signal to read from the local_run_results folder.
config_and_files = get_config_and_results_for_offline_runs(
train_config)
result_files = config_and_files.files
# One file for VAL and one for TRAIN for each child run
assert len(
result_files
) == train_config.get_total_number_of_cross_validation_runs() * 2
for file in result_files:
assert file.execution_mode != ModelExecutionMode.TEST
assert file.dataset_csv_file is not None
assert file.dataset_csv_file.exists()
assert file.metrics_file is not None
assert file.metrics_file.exists()
def test_download_azureml_dataset(
test_output_dirs: OutputFolderForTests) -> None:
dataset_name = "test-dataset"
config = ModelConfigBase(should_validate=False)
azure_config = get_default_azure_config()
runner = MLRunner(config, azure_config)
runner.project_root = test_output_dirs.root_dir
# If the model has neither local_dataset or azure_dataset_id, mount_or_download_dataset should fail.
with pytest.raises(ValueError):
runner.mount_or_download_dataset()
# Pointing the model to a dataset folder that does not exist should raise an Exception
fake_folder = runner.project_root / "foo"
runner.model_config.local_dataset = fake_folder
with pytest.raises(FileNotFoundError):
runner.mount_or_download_dataset()
# If the local dataset folder exists, mount_or_download_dataset should not do anything.
fake_folder.mkdir()
local_dataset = runner.mount_or_download_dataset()
assert local_dataset == fake_folder
# Pointing the model to a dataset in Azure should trigger a download
runner.model_config.local_dataset = None
runner.model_config.azure_dataset_id = dataset_name
with logging_section("Starting download"):
result_path = runner.mount_or_download_dataset()
# Download goes into <project_root> / "datasets" / "test_dataset"
expected_path = runner.project_root / fixed_paths.DATASETS_DIR_NAME / dataset_name
assert result_path == expected_path
assert result_path.is_dir()
dataset_csv = Path(result_path) / DATASET_CSV_FILE_NAME
assert dataset_csv.is_file()
# Check that each individual file in the dataset is present
for folder in [1, *range(10, 20)]:
sub_folder = result_path / str(folder)
sub_folder.is_dir()
for file in [
"ct", "esophagus", "heart", "lung_l", "lung_r", "spinalcord"
]:
f = (sub_folder / file).with_suffix(".nii.gz")
assert f.is_file()
def test_register_model_invalid() -> None:
ws = get_default_workspace()
config = get_model_loader().create_model_config_from_name("Lung")
with pytest.raises(Exception):
ml_runner = MLRunner(config, None)
ml_runner.register_segmentation_model(
run=Run.get_context(),
workspace=ws,
best_epoch=0,
best_epoch_dice=0,
checkpoint_paths=checkpoint_paths,
model_proc=ModelProcessing.DEFAULT)
with pytest.raises(Exception):
ml_runner = MLRunner(config, get_default_azure_config())
ml_runner.register_segmentation_model(
best_epoch=0,
best_epoch_dice=0,
checkpoint_paths=checkpoint_paths,
model_proc=ModelProcessing.DEFAULT)
def test_runner_restart(test_output_dirs: OutputFolderForTests) -> None:
"""
Test if starting training from a folder where the checkpoints folder already has recovery checkpoints picks up
that it is a recovery run. Also checks that we update the start epoch in the config at loading time.
"""
model_config = DummyClassification()
model_config.set_output_to(test_output_dirs.root_dir)
model_config.num_epochs = FIXED_EPOCH + 2
# We save all checkpoints - if recovery works as expected we should have a new checkpoint for epoch 4, 5.
model_config.recovery_checkpoint_save_interval = 1
model_config.recovery_checkpoints_save_last_k = -1
runner = MLRunner(model_config=model_config)
runner.setup(use_mount_or_download_dataset=False)
# Epochs are 0 based for saving
create_model_and_store_checkpoint(model_config,
runner.container.checkpoint_folder /
f"{RECOVERY_CHECKPOINT_FILE_NAME}_epoch="
f"{FIXED_EPOCH - 1}{CHECKPOINT_SUFFIX}",
weights_only=False)
azure_config = get_default_azure_config()
checkpoint_handler = CheckpointHandler(
azure_config=azure_config,
container=runner.container,
project_root=test_output_dirs.root_dir)
_, storing_logger = model_train(checkpoint_handler=checkpoint_handler,
container=runner.container)
# We expect to have 4 checkpoints, FIXED_EPOCH (recovery), FIXED_EPOCH+1, FIXED_EPOCH and best.
assert len(os.listdir(runner.container.checkpoint_folder)) == 4
assert (runner.container.checkpoint_folder /
f"{RECOVERY_CHECKPOINT_FILE_NAME}_epoch="
f"{FIXED_EPOCH - 1}{CHECKPOINT_SUFFIX}").exists()
assert (runner.container.checkpoint_folder /
f"{RECOVERY_CHECKPOINT_FILE_NAME}_epoch="
f"{FIXED_EPOCH}{CHECKPOINT_SUFFIX}").exists()
assert (runner.container.checkpoint_folder /
f"{RECOVERY_CHECKPOINT_FILE_NAME}_epoch="
f"{FIXED_EPOCH + 1}{CHECKPOINT_SUFFIX}").exists()
assert (runner.container.checkpoint_folder /
BEST_CHECKPOINT_FILE_NAME_WITH_SUFFIX).exists()
# Check that we really restarted epoch from epoch FIXED_EPOCH.
assert list(storing_logger.epochs) == [FIXED_EPOCH,
FIXED_EPOCH + 1] # type: ignore
def test_run_ml_with_multi_label_sequence_in_crossval(test_output_dirs: TestOutputDirectories) -> None:
"""
Test training and testing of sequence models that predicts at multiple time points,
including aggregation of cross validation results.
"""
logging_to_stdout()
config = ToyMultiLabelSequenceModel(should_validate=False)
assert config.get_target_indices() == [1, 2, 3]
expected_prediction_targets = ["Seq_pos 01", "Seq_pos 02", "Seq_pos 03"]
target_indices = config.get_target_indices()
assert target_indices
assert len(target_indices) == len(expected_prediction_targets)
config.set_output_to(test_output_dirs.root_dir)
config.dataset_data_frame = _get_multi_label_sequence_dataframe()
config.pre_process_dataset_dataframe()
config.num_epochs = 1
config.number_of_cross_validation_splits = 2
azure_config = get_default_azure_config()
azure_config.train = True
MLRunner(config, azure_config).run()
def test_model_inference_on_single_run(test_output_dirs: OutputFolderForTests) -> None:
falllback_run_id = FALLBACK_HELLO_CONTAINER_RUN
files_to_check = ["test_mse.txt", "test_mae.txt"]
training_run = get_most_recent_run(fallback_run_id_for_local_execution=falllback_run_id)
all_training_files = training_run.get_file_names()
for file in files_to_check:
assert f"outputs/{file}" in all_training_files, f"{file} is missing"
training_folder = test_output_dirs.root_dir / "training"
training_folder.mkdir()
training_files = [training_folder / file for file in files_to_check]
for file, download_path in zip(files_to_check, training_files):
training_run.download_file(f"outputs/{file}", output_file_path=str(download_path))
container = HelloContainer()
container.set_output_to(test_output_dirs.root_dir)
container.model_id = get_most_recent_model_id(fallback_run_id_for_local_execution=falllback_run_id)
azure_config = get_default_azure_config()
azure_config.train = False
ml_runner = MLRunner(container=container, azure_config=azure_config, project_root=test_output_dirs.root_dir)
ml_runner.setup()
ml_runner.run()
inference_files = [container.outputs_folder / file for file in files_to_check]
for inference_file in inference_files:
assert inference_file.exists(), f"{inference_file} is missing"
for training_file, inference_file in zip(training_files, inference_files):
training_lines = training_file.read_text().splitlines()
inference_lines = inference_file.read_text().splitlines()
# We expect all the files we are reading to have a single float value
assert len(training_lines) == 1
train_value = float(training_lines[0].strip())
assert len(inference_lines) == 1
inference_value = float(inference_lines[0].strip())
assert inference_value == pytest.approx(train_value, 1e-6)
def test_register_and_score_model(is_ensemble: bool,
dataset_expected_spacing_xyz: Any,
model_outside_package: bool,
test_output_dirs: OutputFolderForTests) -> None:
"""
End-to-end test which ensures the scoring pipeline is functioning as expected by performing the following:
1) Registering a pre-trained model to AML
2) Checking that a model zip from the registered model can be created successfully
3) Calling the scoring pipeline to check inference can be run from the published model successfully
"""
ws = get_default_workspace()
# Get an existing config as template
loader = get_model_loader("Tests.ML.configs" if model_outside_package else None)
config: SegmentationModelBase = loader.create_model_config_from_name(
model_name="BasicModel2EpochsOutsidePackage" if model_outside_package else "BasicModel2Epochs"
)
config.dataset_expected_spacing_xyz = dataset_expected_spacing_xyz
config.set_output_to(test_output_dirs.root_dir)
# copy checkpoints into the outputs (simulating a run)
stored_checkpoints = full_ml_test_data_path(os.path.join("train_and_test_data", "checkpoints"))
shutil.copytree(str(stored_checkpoints), str(config.checkpoint_folder))
paths = [config.checkpoint_folder / "1_checkpoint.pth.tar"]
checkpoints = paths * 2 if is_ensemble else paths
model = None
model_path = None
# Mocking to get the source from the current directory
# the score.py and python_wrapper.py cannot be moved inside the InnerEye package, which will be the
# only code running (if these tests are run on the package).
with mock.patch('InnerEye.Common.fixed_paths.repository_root_directory',
return_value=tests_root_directory().parent):
try:
tags = {"model_name": config.model_name}
azure_config = get_default_azure_config()
if model_outside_package:
azure_config.extra_code_directory = "Tests" # contains DummyModel
deployment_hook = lambda cfg, azure_cfg, mdl, is_ens: (Path(cfg.model_name), azure_cfg.docker_shm_size)
ml_runner = MLRunner(config, azure_config, model_deployment_hook=deployment_hook)
model, deployment_path, deployment_details = ml_runner.register_segmentation_model(
workspace=ws,
tags=tags,
best_epoch=0,
best_epoch_dice=0,
checkpoint_paths=checkpoints,
model_proc=ModelProcessing.DEFAULT)
assert model is not None
model_path = Path(model.get_model_path(model.name, model.version, ws))
assert (model_path / fixed_paths.ENVIRONMENT_YAML_FILE_NAME).exists()
assert (model_path / Path("InnerEye/ML/runner.py")).exists()
assert deployment_path == Path(config.model_name)
assert deployment_details == azure_config.docker_shm_size
# move test data into the data folder to simulate an actual run
train_and_test_data_dir = full_ml_test_data_path("train_and_test_data")
img_channel_1_name = "id1_channel1.nii.gz"
img_channel_1_path = train_and_test_data_dir / img_channel_1_name
img_channel_2_name = "id1_channel2.nii.gz"
img_channel_2_path = train_and_test_data_dir / img_channel_2_name
# download the registered model and test that we can run the score pipeline on it
model_root = Path(model.download(str(test_output_dirs.root_dir)))
# create a dummy datastore to store model checkpoints and image data
# this simulates the code shapshot being executed in a real run
test_datastore = test_output_dirs.root_dir / "test_datastore"
shutil.move(
str(model_root / "test_outputs"),
str(test_datastore / RELATIVE_TEST_OUTPUTS_PATH)
)
data_root = test_datastore / DEFAULT_DATA_FOLDER
os.makedirs(data_root)
shutil.copy(str(img_channel_1_path), data_root)
shutil.copy(str(img_channel_2_path), data_root)
# run score pipeline as a separate process using the python_wrapper.py code to simulate a real run
return_code = SubprocessConfig(process="python", args=[
str(model_root / "python_wrapper.py"),
"--spawnprocess=python",
str(model_root / "score.py"),
f"--data-folder={str(test_datastore)}",
f"--test_image_channels={img_channel_1_name},{img_channel_2_name}",
"--use_gpu=False"
]).spawn_and_monitor_subprocess()
# check that the process completed as expected
assert return_code == 0
expected_segmentation_path = Path(model_root) / DEFAULT_RESULT_IMAGE_NAME
assert expected_segmentation_path.exists()
# sanity check the resulting segmentation
expected_shape = get_nifti_shape(img_channel_1_path)
image_header = get_unit_image_header()
assert_nifti_content(str(expected_segmentation_path), expected_shape, image_header, [0], np.ubyte)
finally:
# delete the registered model, and any downloaded artifacts
shutil.rmtree(test_output_dirs.root_dir)
if model and model_path:
model.delete()
shutil.rmtree(model_path)
