def test_parsing_with_custom_yaml( test_output_dirs: OutputFolderForTests) -> None: """ Test if additional model or Azure config settings can be read correctly from YAML files. """ yaml_file = test_output_dirs.root_dir / "custom.yml" yaml_file.write_text("""variables: tenant_id: 'foo' start_epoch: 7 random_seed: 1 """) # Arguments partly to be set in AzureConfig, and partly in model config. args = [ "", "--tenant_id=bar", "--model", "Lung", "--num_epochs", "42", "--random_seed", "2" ] with mock.patch("sys.argv", args): runner = Runner(project_root=fixed_paths.repository_root_directory(), yaml_config_file=yaml_file) loader_result = runner.parse_and_load_model() assert loader_result is not None assert runner.azure_config is not None # This is only present in yaml # This is present in yaml and command line, and the latter should be used. assert runner.azure_config.tenant_id == "bar" # Settings in model config: start_epoch is only in yaml assert runner.model_config.start_epoch == 7 # Settings in model config: num_epochs is only on commandline assert runner.model_config.num_epochs == 42 # Settings in model config: random_seed is both in yaml and command line, the latter should be used assert runner.model_config.get_effective_random_seed() == 2 assert loader_result.overrides == {"num_epochs": 42, "random_seed": 2}
def test_cross_validation_for_lighting_container_models_is_supported() -> None: """ Prior to https://github.com/microsoft/InnerEye-DeepLearning/pull/483 we raised an exception in runner.run when cross validation was attempted on a lightning container. This test checks that we do not raise the exception anymore, and instead pass on a cross validation hyperdrive config to azure_runner's submit_to_azureml method. """ args_list = [ "--model=HelloContainer", "--number_of_cross_validation_splits=5", "--azureml=True" ] mock_run = mock.MagicMock() mock_run.id = "foo" mock_run.experiment.name = "bar" with mock.patch("sys.argv", [""] + args_list): runner = Runner(project_root=fixed_paths.repository_root_directory(), yaml_config_file=fixed_paths.SETTINGS_YAML_FILE) with mock.patch( "health_azure.himl.submit_run", return_value=mock_run) as create_and_submit_experiment_patch: with pytest.raises(SystemExit): runner.run() assert runner.lightning_container.model_name == 'HelloContainer' assert runner.lightning_container.number_of_cross_validation_splits == 5 script_run_config_arg = create_and_submit_experiment_patch.call_args[ 1]["script_run_config"] assert isinstance(script_run_config_arg, HyperDriveConfig)
def test_create_ml_runner_args(is_default_namespace: bool, test_output_dirs: TestOutputDirectories, is_offline_run: bool) -> None: """Test round trip parsing of commandline arguments: From arguments to the Azure runner to the arguments of the ML runner, checking that whatever is passed on can be correctly parsed.""" logging_to_stdout() model_name = "Lung" outputs_folder = Path(test_output_dirs.root_dir) project_root = fixed_paths.repository_root_directory() if is_default_namespace: model_configs_namespace = None else: model_configs_namespace = "Tests.ML.configs" model_name = "DummyModel" args_list = [ f"--model={model_name}", "--train=True", "--l_rate=100.0", "--norm_method=Simple Norm", "--subscription_id", "Test1", "--tenant_id=Test2", "--application_id", "Test3", "--datasets_storage_account=Test4", "--datasets_container", "Test5", "--pytest_mark", "gpu", f"--output_to={outputs_folder}" ] if not is_default_namespace: args_list.append( f"--model_configs_namespace={model_configs_namespace}") with mock.patch("sys.argv", [""] + args_list): with mock.patch( "InnerEye.ML.deep_learning_config.is_offline_run_context", return_value=is_offline_run): runner = Runner(project_root=project_root, yaml_config_file=fixed_paths.SETTINGS_YAML_FILE) runner.parse_and_load_model() azure_config = runner.azure_config model_config = runner.model_config assert azure_config.datasets_storage_account == "Test4" assert azure_config.model == model_name assert model_config.l_rate == 100.0 assert model_config.norm_method == PhotometricNormalizationMethod.SimpleNorm if is_offline_run: # The actual output folder must be a subfolder of the folder given on the commandline. The folder will contain # a timestamp, that will start with the year number, hence will start with 20... assert str(model_config.outputs_folder).startswith( str(outputs_folder / "20")) assert model_config.logs_folder == (model_config.outputs_folder / DEFAULT_LOGS_DIR_NAME) else: # For runs inside AzureML, the output folder is the project root (the root of the folders that are # included in the snapshot). The "outputs_to" argument will be ignored. assert model_config.outputs_folder == (project_root / DEFAULT_AML_UPLOAD_DIR) assert model_config.logs_folder == (project_root / DEFAULT_LOGS_DIR_NAME) assert not hasattr(model_config, "datasets_storage_account") assert azure_config.pytest_mark == "gpu"
def default_runner() -> Runner: """ Create an InnerEye Runner object with the default settings, pointing to the repository root and default settings files. """ return Runner(project_root=repository_root_directory(), yaml_config_file=fixed_paths.SETTINGS_YAML_FILE)
def test_read_yaml_file_into_args( test_output_dirs: TestOutputDirectories) -> None: """ Test if the arguments for specifying the YAML config file with storage account, etc are correctly wired up. """ empty_yaml = Path(test_output_dirs.root_dir) / "nothing.yaml" empty_yaml.write_text("variables:\n") with mock.patch("sys.argv", ["", "--model=Lung"]): # Default behaviour: Application ID (service principal) should be picked up from YAML runner1 = Runner(project_root=fixed_paths.repository_root_directory(), yaml_config_file=fixed_paths.SETTINGS_YAML_FILE) runner1.parse_and_load_model() assert len(runner1.azure_config.application_id) > 0 # When specifying a dummy YAML file that does not contain the application ID, it should not # be set. runner2 = Runner(project_root=fixed_paths.repository_root_directory(), yaml_config_file=empty_yaml) runner2.parse_and_load_model() assert runner2.azure_config.application_id == ""
def test_read_yaml_file_into_args(test_output_dirs: OutputFolderForTests) -> None: """ Test if the arguments for specifying the YAML config file with storage account, etc are correctly wired up. """ empty_yaml = test_output_dirs.root_dir / "nothing.yaml" empty_yaml.write_text("variables:\n") with mock.patch("sys.argv", ["", "--model=Lung"]): # Default behaviour: tenant_id should be picked up from YAML runner1 = Runner(project_root=fixed_paths.repository_root_directory(), yaml_config_file=fixed_paths.SETTINGS_YAML_FILE) runner1.parse_and_load_model() assert len(runner1.azure_config.application_id) > 0 assert len(runner1.azure_config.resource_group) > 0 # When specifying a dummy YAML file that does not contain any settings, no information in AzureConfig should # be set. Some settings are read from a private settings file, most notably application ID, which should # be present on people's local dev boxes. Hence, only assert on `resource_group` here. runner2 = Runner(project_root=fixed_paths.repository_root_directory(), yaml_config_file=empty_yaml) runner2.parse_and_load_model() assert runner2.azure_config.resource_group == ""
def run(self) -> None: """ Driver function to run a ML experiment. If an offline cross validation run is requested, then this function is recursively called for each cross validation split. """ if self.is_offline_cross_val_parent_run(): if self.model_config.is_segmentation_model: raise NotImplementedError( "Offline cross validation is only supported for classification models." ) self.spawn_offline_cross_val_classification_child_runs() return # Get the AzureML context in which the script is running if not self.model_config.is_offline_run and PARENT_RUN_CONTEXT is not None: logging.info("Setting tags from parent run.") self.set_run_tags_from_parent() self.save_build_info_for_dotnet_consumers() # Set data loader start method self.set_multiprocessing_start_method() # configure recovery container if provided checkpoint_handler = CheckpointHandler(model_config=self.model_config, azure_config=self.azure_config, project_root=self.project_root, run_context=RUN_CONTEXT) checkpoint_handler.discover_and_download_checkpoints_from_previous_runs( ) # do training and inference, unless the "only register" switch is set (which requires a run_recovery # to be valid). if not self.azure_config.register_model_only_for_epoch: # Set local_dataset to the mounted path specified in azure_runner.py, if any, or download it if that fails # and config.local_dataset was not already set. self.model_config.local_dataset = self.mount_or_download_dataset() self.model_config.write_args_file() logging.info(str(self.model_config)) # Ensure that training runs are fully reproducible - setting random seeds alone is not enough! make_pytorch_reproducible() # Check for existing dataset.csv file in the correct locations. Skip that if a dataset has already been # loaded (typically only during tests) if self.model_config.dataset_data_frame is None: assert self.model_config.local_dataset is not None ml_util.validate_dataset_paths(self.model_config.local_dataset) # train a new model if required if self.azure_config.train: with logging_section("Model training"): model_train(self.model_config, checkpoint_handler) else: self.model_config.write_dataset_files() self.create_activation_maps() # log the number of epochs used for model training RUN_CONTEXT.log(name="Train epochs", value=self.model_config.num_epochs) # We specify the ModelProcessing as DEFAULT here even if the run_recovery points to an ensemble run, because # the current run is a single one. See the documentation of ModelProcessing for more details. best_epoch = self.run_inference_and_register_model( checkpoint_handler, ModelProcessing.DEFAULT) # Generate report if best_epoch: Runner.generate_report(self.model_config, best_epoch, ModelProcessing.DEFAULT) elif self.model_config.is_scalar_model and len( self.model_config.get_test_epochs()) == 1: # We don't register scalar models but still want to create a report if we have run inference. Runner.generate_report(self.model_config, self.model_config.get_test_epochs()[0], ModelProcessing.DEFAULT)
def test_create_ml_runner_args(is_container: bool, test_output_dirs: OutputFolderForTests, is_offline_run: bool, set_output_to: bool, tmpdir: Path) -> None: """Test round trip parsing of commandline arguments: From arguments to the Azure runner to the arguments of the ML runner, checking that whatever is passed on can be correctly parsed. It also checks that the output files go into the right place in local runs and in AzureML.""" logging_to_stdout() model_name = "DummyContainerWithPlainLightning" if is_container else "DummyModel" if is_container: dataset_folder = tmpdir else: local_dataset = DummyModel().local_dataset assert local_dataset is not None assert local_dataset.is_dir() dataset_folder = local_dataset outputs_folder = test_output_dirs.root_dir project_root = fixed_paths.repository_root_directory() model_configs_namespace = "Tests.ML.configs" args_list = [ f"--model={model_name}", "--train=True", "--l_rate=100.0", "--subscription_id", "Test1", "--tenant_id=Test2", "--application_id", "Test3", "--azureml_datastore", "Test5" ] # toggle the output_to flag off only for online runs if set_output_to or is_offline_run: args_list.append(f"--output_to={outputs_folder}") if not is_container: args_list.append("--norm_method=Simple Norm") args_list.append(f"--model_configs_namespace={model_configs_namespace}") with mock.patch("sys.argv", [""] + args_list): with mock.patch( "InnerEye.ML.deep_learning_config.is_offline_run_context", return_value=is_offline_run): with mock.patch("InnerEye.ML.run_ml.MLRunner.run", return_value=None): runner = Runner( project_root=project_root, yaml_config_file=fixed_paths.SETTINGS_YAML_FILE) runner.parse_and_load_model() azure_run_info = AzureRunInfo(input_datasets=[dataset_folder], output_datasets=[None], run=None, is_running_in_azure_ml=False, output_folder=Path.cwd(), logs_folder=Path.cwd(), mount_contexts=[]) # Only when calling config.create_filesystem we expect to see the correct paths, and this happens # inside run_in_situ runner.run_in_situ(azure_run_info) azure_config = runner.azure_config container_or_legacy_config = runner.lightning_container if is_container else runner.model_config assert azure_config.model == model_name assert container_or_legacy_config is not None if not is_container: assert isinstance(container_or_legacy_config, DeepLearningConfig) assert container_or_legacy_config.norm_method == PhotometricNormalizationMethod.SimpleNorm if set_output_to or is_offline_run: # The actual output folder must be a subfolder of the folder given on the commandline. The folder will contain # a timestamp, that will start with the year number, hence will start with 20... assert str(container_or_legacy_config.outputs_folder).startswith( str(outputs_folder / "20")) assert container_or_legacy_config.logs_folder == \ (container_or_legacy_config.outputs_folder / DEFAULT_LOGS_DIR_NAME) else: # For runs inside AzureML, the output folder is the project root (the root of the folders that are # included in the snapshot). The "outputs_to" argument will be ignored. assert container_or_legacy_config.outputs_folder == ( project_root / DEFAULT_AML_UPLOAD_DIR) assert container_or_legacy_config.logs_folder == ( project_root / DEFAULT_LOGS_DIR_NAME) assert not hasattr(container_or_legacy_config, "azureml_datastore") # Container setup should copy the path of the local dataset from AzureRunInfo to the local_dataset field assert container_or_legacy_config.local_dataset == dataset_folder