def test_parsing_with_custom_yaml(
test_output_dirs: OutputFolderForTests) -> None:
"""
Test if additional model or Azure config settings can be read correctly from YAML files.
"""
yaml_file = test_output_dirs.root_dir / "custom.yml"
yaml_file.write_text("""variables:
tenant_id: 'foo'
start_epoch: 7
random_seed: 1
""")
# Arguments partly to be set in AzureConfig, and partly in model config.
args = [
"", "--tenant_id=bar", "--model", "Lung", "--num_epochs", "42",
"--random_seed", "2"
]
with mock.patch("sys.argv", args):
runner = Runner(project_root=fixed_paths.repository_root_directory(),
yaml_config_file=yaml_file)
loader_result = runner.parse_and_load_model()
assert loader_result is not None
assert runner.azure_config is not None
# This is only present in yaml
# This is present in yaml and command line, and the latter should be used.
assert runner.azure_config.tenant_id == "bar"
# Settings in model config: start_epoch is only in yaml
assert runner.model_config.start_epoch == 7
# Settings in model config: num_epochs is only on commandline
assert runner.model_config.num_epochs == 42
# Settings in model config: random_seed is both in yaml and command line, the latter should be used
assert runner.model_config.get_effective_random_seed() == 2
assert loader_result.overrides == {"num_epochs": 42, "random_seed": 2}
def test_cross_validation_for_lighting_container_models_is_supported() -> None:
"""
Prior to https://github.com/microsoft/InnerEye-DeepLearning/pull/483 we raised an exception in
runner.run when cross validation was attempted on a lightning container. This test checks that
we do not raise the exception anymore, and instead pass on a cross validation hyperdrive config
to azure_runner's submit_to_azureml method.
"""
args_list = [
"--model=HelloContainer", "--number_of_cross_validation_splits=5",
"--azureml=True"
]
mock_run = mock.MagicMock()
mock_run.id = "foo"
mock_run.experiment.name = "bar"
with mock.patch("sys.argv", [""] + args_list):
runner = Runner(project_root=fixed_paths.repository_root_directory(),
yaml_config_file=fixed_paths.SETTINGS_YAML_FILE)
with mock.patch(
"health_azure.himl.submit_run",
return_value=mock_run) as create_and_submit_experiment_patch:
with pytest.raises(SystemExit):
runner.run()
assert runner.lightning_container.model_name == 'HelloContainer'
assert runner.lightning_container.number_of_cross_validation_splits == 5
script_run_config_arg = create_and_submit_experiment_patch.call_args[
1]["script_run_config"]
assert isinstance(script_run_config_arg, HyperDriveConfig)
def test_create_ml_runner_args(is_default_namespace: bool,
test_output_dirs: TestOutputDirectories,
is_offline_run: bool) -> None:
"""Test round trip parsing of commandline arguments:
From arguments to the Azure runner to the arguments of the ML runner, checking that
whatever is passed on can be correctly parsed."""
logging_to_stdout()
model_name = "Lung"
outputs_folder = Path(test_output_dirs.root_dir)
project_root = fixed_paths.repository_root_directory()
if is_default_namespace:
model_configs_namespace = None
else:
model_configs_namespace = "Tests.ML.configs"
model_name = "DummyModel"
args_list = [
f"--model={model_name}", "--train=True", "--l_rate=100.0",
"--norm_method=Simple Norm", "--subscription_id", "Test1",
"--tenant_id=Test2", "--application_id", "Test3",
"--datasets_storage_account=Test4", "--datasets_container", "Test5",
"--pytest_mark", "gpu", f"--output_to={outputs_folder}"
]
if not is_default_namespace:
args_list.append(
f"--model_configs_namespace={model_configs_namespace}")
with mock.patch("sys.argv", [""] + args_list):
with mock.patch(
"InnerEye.ML.deep_learning_config.is_offline_run_context",
return_value=is_offline_run):
runner = Runner(project_root=project_root,
yaml_config_file=fixed_paths.SETTINGS_YAML_FILE)
runner.parse_and_load_model()
azure_config = runner.azure_config
model_config = runner.model_config
assert azure_config.datasets_storage_account == "Test4"
assert azure_config.model == model_name
assert model_config.l_rate == 100.0
assert model_config.norm_method == PhotometricNormalizationMethod.SimpleNorm
if is_offline_run:
# The actual output folder must be a subfolder of the folder given on the commandline. The folder will contain
# a timestamp, that will start with the year number, hence will start with 20...
assert str(model_config.outputs_folder).startswith(
str(outputs_folder / "20"))
assert model_config.logs_folder == (model_config.outputs_folder /
DEFAULT_LOGS_DIR_NAME)
else:
# For runs inside AzureML, the output folder is the project root (the root of the folders that are
# included in the snapshot). The "outputs_to" argument will be ignored.
assert model_config.outputs_folder == (project_root /
DEFAULT_AML_UPLOAD_DIR)
assert model_config.logs_folder == (project_root /
DEFAULT_LOGS_DIR_NAME)
assert not hasattr(model_config, "datasets_storage_account")
assert azure_config.pytest_mark == "gpu"
def default_runner() -> Runner:
"""
Create an InnerEye Runner object with the default settings, pointing to the repository root and
default settings files.
"""
return Runner(project_root=repository_root_directory(),
yaml_config_file=fixed_paths.SETTINGS_YAML_FILE)
def test_read_yaml_file_into_args(
test_output_dirs: TestOutputDirectories) -> None:
"""
Test if the arguments for specifying the YAML config file with storage account, etc
are correctly wired up.
"""
empty_yaml = Path(test_output_dirs.root_dir) / "nothing.yaml"
empty_yaml.write_text("variables:\n")
with mock.patch("sys.argv", ["", "--model=Lung"]):
# Default behaviour: Application ID (service principal) should be picked up from YAML
runner1 = Runner(project_root=fixed_paths.repository_root_directory(),
yaml_config_file=fixed_paths.SETTINGS_YAML_FILE)
runner1.parse_and_load_model()
assert len(runner1.azure_config.application_id) > 0
# When specifying a dummy YAML file that does not contain the application ID, it should not
# be set.
runner2 = Runner(project_root=fixed_paths.repository_root_directory(),
yaml_config_file=empty_yaml)
runner2.parse_and_load_model()
assert runner2.azure_config.application_id == ""
def test_read_yaml_file_into_args(test_output_dirs: OutputFolderForTests) -> None:
"""
Test if the arguments for specifying the YAML config file with storage account, etc
are correctly wired up.
"""
empty_yaml = test_output_dirs.root_dir / "nothing.yaml"
empty_yaml.write_text("variables:\n")
with mock.patch("sys.argv", ["", "--model=Lung"]):
# Default behaviour: tenant_id should be picked up from YAML
runner1 = Runner(project_root=fixed_paths.repository_root_directory(),
yaml_config_file=fixed_paths.SETTINGS_YAML_FILE)
runner1.parse_and_load_model()
assert len(runner1.azure_config.application_id) > 0
assert len(runner1.azure_config.resource_group) > 0
# When specifying a dummy YAML file that does not contain any settings, no information in AzureConfig should
# be set. Some settings are read from a private settings file, most notably application ID, which should
# be present on people's local dev boxes. Hence, only assert on `resource_group` here.
runner2 = Runner(project_root=fixed_paths.repository_root_directory(),
yaml_config_file=empty_yaml)
runner2.parse_and_load_model()
assert runner2.azure_config.resource_group == ""
def run(self) -> None:
"""
Driver function to run a ML experiment. If an offline cross validation run is requested, then
this function is recursively called for each cross validation split.
"""
if self.is_offline_cross_val_parent_run():
if self.model_config.is_segmentation_model:
raise NotImplementedError(
"Offline cross validation is only supported for classification models."
)
self.spawn_offline_cross_val_classification_child_runs()
return
# Get the AzureML context in which the script is running
if not self.model_config.is_offline_run and PARENT_RUN_CONTEXT is not None:
logging.info("Setting tags from parent run.")
self.set_run_tags_from_parent()
self.save_build_info_for_dotnet_consumers()
# Set data loader start method
self.set_multiprocessing_start_method()
# configure recovery container if provided
checkpoint_handler = CheckpointHandler(model_config=self.model_config,
azure_config=self.azure_config,
project_root=self.project_root,
run_context=RUN_CONTEXT)
checkpoint_handler.discover_and_download_checkpoints_from_previous_runs(
)
# do training and inference, unless the "only register" switch is set (which requires a run_recovery
# to be valid).
if not self.azure_config.register_model_only_for_epoch:
# Set local_dataset to the mounted path specified in azure_runner.py, if any, or download it if that fails
# and config.local_dataset was not already set.
self.model_config.local_dataset = self.mount_or_download_dataset()
self.model_config.write_args_file()
logging.info(str(self.model_config))
# Ensure that training runs are fully reproducible - setting random seeds alone is not enough!
make_pytorch_reproducible()
# Check for existing dataset.csv file in the correct locations. Skip that if a dataset has already been
# loaded (typically only during tests)
if self.model_config.dataset_data_frame is None:
assert self.model_config.local_dataset is not None
ml_util.validate_dataset_paths(self.model_config.local_dataset)
# train a new model if required
if self.azure_config.train:
with logging_section("Model training"):
model_train(self.model_config, checkpoint_handler)
else:
self.model_config.write_dataset_files()
self.create_activation_maps()
# log the number of epochs used for model training
RUN_CONTEXT.log(name="Train epochs",
value=self.model_config.num_epochs)
# We specify the ModelProcessing as DEFAULT here even if the run_recovery points to an ensemble run, because
# the current run is a single one. See the documentation of ModelProcessing for more details.
best_epoch = self.run_inference_and_register_model(
checkpoint_handler, ModelProcessing.DEFAULT)
# Generate report
if best_epoch:
Runner.generate_report(self.model_config, best_epoch,
ModelProcessing.DEFAULT)
elif self.model_config.is_scalar_model and len(
self.model_config.get_test_epochs()) == 1:
# We don't register scalar models but still want to create a report if we have run inference.
Runner.generate_report(self.model_config,
self.model_config.get_test_epochs()[0],
ModelProcessing.DEFAULT)
def test_create_ml_runner_args(is_container: bool,
test_output_dirs: OutputFolderForTests,
is_offline_run: bool, set_output_to: bool,
tmpdir: Path) -> None:
"""Test round trip parsing of commandline arguments:
From arguments to the Azure runner to the arguments of the ML runner, checking that
whatever is passed on can be correctly parsed. It also checks that the output files go into the right place
in local runs and in AzureML."""
logging_to_stdout()
model_name = "DummyContainerWithPlainLightning" if is_container else "DummyModel"
if is_container:
dataset_folder = tmpdir
else:
local_dataset = DummyModel().local_dataset
assert local_dataset is not None
assert local_dataset.is_dir()
dataset_folder = local_dataset
outputs_folder = test_output_dirs.root_dir
project_root = fixed_paths.repository_root_directory()
model_configs_namespace = "Tests.ML.configs"
args_list = [
f"--model={model_name}", "--train=True", "--l_rate=100.0",
"--subscription_id", "Test1", "--tenant_id=Test2", "--application_id",
"Test3", "--azureml_datastore", "Test5"
]
# toggle the output_to flag off only for online runs
if set_output_to or is_offline_run:
args_list.append(f"--output_to={outputs_folder}")
if not is_container:
args_list.append("--norm_method=Simple Norm")
args_list.append(f"--model_configs_namespace={model_configs_namespace}")
with mock.patch("sys.argv", [""] + args_list):
with mock.patch(
"InnerEye.ML.deep_learning_config.is_offline_run_context",
return_value=is_offline_run):
with mock.patch("InnerEye.ML.run_ml.MLRunner.run",
return_value=None):
runner = Runner(
project_root=project_root,
yaml_config_file=fixed_paths.SETTINGS_YAML_FILE)
runner.parse_and_load_model()
azure_run_info = AzureRunInfo(input_datasets=[dataset_folder],
output_datasets=[None],
run=None,
is_running_in_azure_ml=False,
output_folder=Path.cwd(),
logs_folder=Path.cwd(),
mount_contexts=[])
# Only when calling config.create_filesystem we expect to see the correct paths, and this happens
# inside run_in_situ
runner.run_in_situ(azure_run_info)
azure_config = runner.azure_config
container_or_legacy_config = runner.lightning_container if is_container else runner.model_config
assert azure_config.model == model_name
assert container_or_legacy_config is not None
if not is_container:
assert isinstance(container_or_legacy_config, DeepLearningConfig)
assert container_or_legacy_config.norm_method == PhotometricNormalizationMethod.SimpleNorm
if set_output_to or is_offline_run:
# The actual output folder must be a subfolder of the folder given on the commandline. The folder will contain
# a timestamp, that will start with the year number, hence will start with 20...
assert str(container_or_legacy_config.outputs_folder).startswith(
str(outputs_folder / "20"))
assert container_or_legacy_config.logs_folder == \
(container_or_legacy_config.outputs_folder / DEFAULT_LOGS_DIR_NAME)
else:
# For runs inside AzureML, the output folder is the project root (the root of the folders that are
# included in the snapshot). The "outputs_to" argument will be ignored.
assert container_or_legacy_config.outputs_folder == (
project_root / DEFAULT_AML_UPLOAD_DIR)
assert container_or_legacy_config.logs_folder == (
project_root / DEFAULT_LOGS_DIR_NAME)
assert not hasattr(container_or_legacy_config, "azureml_datastore")
# Container setup should copy the path of the local dataset from AzureRunInfo to the local_dataset field
assert container_or_legacy_config.local_dataset == dataset_folder
