def test_create_python_env() -> None: """ Checks if environment variables in the SourceConfig are correctly passed through to the Python environment. Environment variables in SourceConfig are only used in the internal InnerEye repo. :return: """ foo = "foo" bar = "bar" entry_script = Path("something.py") conda_file = get_environment_yaml_file() s = SourceConfig(root_folder=Path(""), entry_script=entry_script, conda_dependencies_files=[conda_file], environment_variables={foo: bar}) env = get_or_create_python_environment( source_config=s, azure_config=get_default_azure_config(), register_environment=False) assert foo in env.environment_variables assert env.environment_variables[foo] == bar # Check that some of the basic packages that we expect to always exist are picked up correctly in the Conda env def remove_version_number(items: Iterator[str]) -> Set[str]: return set(c.split("=")[0] for c in items) assert "pytorch" in remove_version_number( env.python.conda_dependencies.conda_packages) assert "pytorch-lightning" in remove_version_number( env.python.conda_dependencies.pip_packages)
def test_run_ml_with_classification_model( test_output_dirs: OutputFolderForTests, number_of_offline_cross_validation_splits: int, model_name: str) -> None: """ Test training and testing of classification models, when it is started together via run_ml. """ logging_to_stdout() azure_config = get_default_azure_config() azure_config.train = True config: ScalarModelBase = ModelConfigLoader[ScalarModelBase]() \ .create_model_config_from_name(model_name) config.number_of_cross_validation_splits = number_of_offline_cross_validation_splits config.set_output_to(test_output_dirs.root_dir) # Trying to run DDP from the test suite hangs, hence restrict to single GPU. config.max_num_gpus = 1 MLRunner(config, azure_config).run() _check_offline_cross_validation_output_files(config) if config.perform_cross_validation: # Test that the result files can be correctly picked up by the cross validation routine. # For that, we point the downloader to the local results folder. The core download method # recognizes run_recovery_id == None as the signal to read from the local_run_results folder. config_and_files = get_config_and_results_for_offline_runs(config) result_files = config_and_files.files # One file for VAL and one for TRAIN for each child run assert len(result_files ) == config.get_total_number_of_cross_validation_runs() * 2 for file in result_files: assert file.execution_mode != ModelExecutionMode.TEST assert file.dataset_csv_file is not None assert file.dataset_csv_file.exists() assert file.metrics_file is not None assert file.metrics_file.exists()
def test_run_ml_with_sequence_model(use_combined_model: bool, imaging_feature_type: ImagingFeatureType, test_output_dirs: TestOutputDirectories) -> None: """ Test training and testing of sequence models, when it is started together via run_ml. """ logging_to_stdout() config = ToySequenceModel(use_combined_model, imaging_feature_type, should_validate=False, sequence_target_positions=[2, 10]) config.set_output_to(test_output_dirs.root_dir) config.dataset_data_frame = _get_mock_sequence_dataset() config.num_epochs = 1 config.max_batch_grad_cam = 1 # make sure we are testing with at least one sequence position that will not exist # to ensure correct handling of sequences that do not contain all the expected target positions assert max(config.sequence_target_positions) > config.dataset_data_frame[config.sequence_column].astype(float).max() # Patch the load_images function that will be called once we access a dataset item image_and_seg = ImageAndSegmentations[np.ndarray](images=np.random.uniform(0, 1, SCAN_SIZE), segmentations=np.random.randint(0, 2, SCAN_SIZE)) with mock.patch('InnerEye.ML.utils.io_util.load_image_in_known_formats', return_value=image_and_seg): azure_config = get_default_azure_config() azure_config.train = True MLRunner(config, azure_config).run()
def test_download_pytest_file(test_output_dirs: OutputFolderForTests) -> None: output_dir = test_output_dirs.root_dir azure_config = get_default_azure_config() workspace = azure_config.get_workspace() def get_run_and_download_pytest(branch: str, number: int) -> Optional[Path]: experiment = Experiment(workspace, name=to_azure_friendly_string(branch)) runs = [run for run in experiment.get_runs() if run.number == number] if len(runs) != 1: raise ValueError( f"Expected to get exactly 1 run in experiment {experiment.name}" ) return download_pytest_result(runs[0], output_dir) # PR 49 is a recent successful build that generated a pytest file. # Run 6 in that experiment was canceled did not yet write the pytest file: with pytest.raises(ValueError) as ex: get_run_and_download_pytest("refs/pull/219/merge", 6) assert "No pytest result file" in str(ex) downloaded = get_run_and_download_pytest("refs/pull/219/merge", 7) assert downloaded is not None assert downloaded.exists() # Delete the file - it should be cleaned up with the test output directories though. # If the file remained, it would be uploaded as a test result file to Azure DevOps downloaded.unlink()
def test_dataset_consumption1() -> None: """ Creating datasets, case 1: Azure datasets given, no local folders or mount points :return: """ azure_config = get_default_azure_config() datasets = create_dataset_configs(azure_config, all_azure_dataset_ids=["1", "2"], all_dataset_mountpoints=[], all_local_datasets=[]) assert len(datasets) == 2 assert datasets[0].name == "1" assert datasets[1].name == "2" for i in range(2): assert datasets[i].local_folder is None assert datasets[i].target_folder is None # Two error cases: number of mount points or number of local datasets does not match with pytest.raises(ValueError) as ex: create_dataset_configs(azure_config, all_azure_dataset_ids=["1", "2"], all_dataset_mountpoints=["mp"], all_local_datasets=[]) assert "Invalid dataset setup" in str(ex) with pytest.raises(ValueError) as ex: create_dataset_configs(azure_config, all_azure_dataset_ids=["1", "2"], all_dataset_mountpoints=[], all_local_datasets=[Path("local")]) assert "Invalid dataset setup" in str(ex)
def test_dataset_consumption5() -> None: """ Test error handling for empty dataset IDs. """ azure_config = get_default_azure_config() with pytest.raises(ValueError) as ex: azure_config.get_or_create_dataset("") assert "No dataset ID provided" in str(ex)
def test_dataset_consumption2() -> None: """ Test that given mount point with empty dataset ID raises an error """ azure_config = get_default_azure_config() with pytest.raises(ValueError) as ex: create_dataset_consumptions(azure_config, [""], ["foo"]) assert "but a mount point has been provided" in str(ex)
def test_dataset_consumption3() -> None: """ Test that a matching number of mount points is created. """ azure_config = get_default_azure_config() assert len( create_dataset_consumptions(azure_config, ["test-dataset", "test-dataset"], [])) == 2
def get_most_recent_run(fallback_run_id_for_local_execution: str = FALLBACK_SINGLE_RUN) -> Run: """ Gets the name of the most recently executed AzureML run, instantiates that Run object and returns it. :param fallback_run_id_for_local_execution: A hardcoded AzureML run ID that is used when executing this code on a local box, outside of Azure build agents. """ run_recovery_id = get_most_recent_run_id(fallback_run_id_for_local_execution=fallback_run_id_for_local_execution) return get_default_azure_config().fetch_run(run_recovery_id=run_recovery_id)
def test_dataset_consumption4() -> None: """ Test error handling for number of mount points. """ azure_config = get_default_azure_config() with pytest.raises(ValueError) as ex: create_dataset_consumptions(azure_config, ["test-dataset", "test-dataset"], ["foo"]) assert "must equal the number of Azure dataset IDs" in str(ex)
def runner_config() -> AzureConfig: """ Gets an Azure config that masks out the storage account for datasets, to avoid accidental overwriting. :return: """ config = get_default_azure_config() config.model = "" config.train = False return config
def test_get_comparison_data(test_output_dirs: TestOutputDirectories) -> None: azure_config = get_default_azure_config() comparison_name = "DefaultName" comparison_path = DEFAULT_RUN_RECOVERY_ID + "/outputs/epoch_002/Test" baselines = get_comparison_baselines(Path(test_output_dirs.root_dir), azure_config, [(comparison_name, comparison_path)]) assert len(baselines) == 1 assert baselines[0].name == comparison_name
def test_download_dataset_via_blobxfer(test_output_dirs: TestOutputDirectories) -> None: azure_config = get_default_azure_config() result_path = run_ml.download_dataset_via_blobxfer(dataset_id="test-dataset", azure_config=azure_config, target_folder=Path(test_output_dirs.root_dir)) assert result_path assert result_path.is_dir() dataset_csv = Path(result_path) / DATASET_CSV_FILE_NAME assert dataset_csv.exists()
def test_dataset_consumption4() -> None: """ Creating datasets, case 4: no datasets at all """ azure_config = get_default_azure_config() datasets = create_dataset_configs(azure_config, all_azure_dataset_ids=[], all_dataset_mountpoints=[], all_local_datasets=[]) assert len(datasets) == 0
def test_get_comparison_data(test_output_dirs: OutputFolderForTests) -> None: azure_config = get_default_azure_config() comparison_name = "DefaultName" comparison_path = get_most_recent_run_id() + \ f"/{DEFAULT_AML_UPLOAD_DIR}/{BEST_EPOCH_FOLDER_NAME}/{ModelExecutionMode.TEST.value}" baselines = get_comparison_baselines(test_output_dirs.root_dir, azure_config, [(comparison_name, comparison_path)]) assert len(baselines) == 1 assert baselines[0].name == comparison_name
def test_dataset_consumption3() -> None: """ Creating datasets, case 3: local datasets only. This should generate no results """ azure_config = get_default_azure_config() datasets = create_dataset_configs(azure_config, all_azure_dataset_ids=[], all_dataset_mountpoints=[], all_local_datasets=[Path("l1"), Path("l2")]) assert len(datasets) == 0
def test_run_ml_with_multi_label_sequence_model( test_output_dirs: OutputFolderForTests) -> None: """ Test training and testing of sequence models that predicts at multiple time points, when it is started via run_ml. """ logging_to_stdout() config = ToyMultiLabelSequenceModel(should_validate=False) assert config.get_target_indices() == [1, 2, 3] expected_prediction_targets = [ f"{SEQUENCE_POSITION_HUE_NAME_PREFIX} {x}" for x in ["01", "02", "03"] ] _target_indices = config.get_target_indices() assert _target_indices is not None assert len(_target_indices) == len(expected_prediction_targets) metrics_dict = create_metrics_dict_for_scalar_models(config) assert metrics_dict.get_hue_names( include_default=False) == expected_prediction_targets config.set_output_to(test_output_dirs.root_dir) # Create a fake dataset directory to make config validation pass config.local_dataset = test_output_dirs.root_dir config.dataset_data_frame = _get_multi_label_sequence_dataframe() config.pre_process_dataset_dataframe() config.num_epochs = 1 config.max_batch_grad_cam = 1 azure_config = get_default_azure_config() azure_config.train = True MLRunner(config, azure_config).run() # The metrics file should have one entry per epoch per subject per prediction target, # for all the 3 prediction targets. metrics_file = config.outputs_folder / "Train" / SUBJECT_METRICS_FILE_NAME assert metrics_file.exists() metrics = pd.read_csv(metrics_file) assert LoggingColumns.Patient.value in metrics assert LoggingColumns.Epoch.value in metrics assert LoggingColumns.Hue.value in metrics assert metrics[LoggingColumns.Hue.value].unique().tolist( ) == expected_prediction_targets group_by_subject = metrics.groupby( by=[LoggingColumns.Patient.value, LoggingColumns.Epoch.value]) expected_prediction_target_lengths = [3, 2, 3, 3] for i, x in enumerate(group_by_subject): assert len(x[1]) == expected_prediction_target_lengths[i] group_by_subject_and_target = metrics.groupby(by=[ LoggingColumns.Patient.value, LoggingColumns.Epoch.value, LoggingColumns.Hue.value ]) for _, group in group_by_subject_and_target: assert len(group) == 1
def test_download_or_get_local_blobs(is_current_run: bool, test_config: PlotCrossValidationConfig, test_output_dirs: OutputFolderForTests) -> None: azure_config = get_default_azure_config() azure_config.get_workspace() assert test_config.run_recovery_id is not None run = Run.get_context() if is_current_run else azure_config.fetch_run(test_config.run_recovery_id) run_outputs_dir = full_ml_test_data_path() if is_current_run else Path(DEFAULT_AML_UPLOAD_DIR) test_config.outputs_directory = run_outputs_dir dst = test_config.download_or_get_local_file( blob_to_download="dataset.csv", destination=test_output_dirs.root_dir, run=run ) assert dst is not None assert dst.exists()
def test_image_encoder_with_segmentation( test_output_dirs: OutputFolderForTests, encode_channels_jointly: bool, aggregation_type: AggregationType, imaging_feature_type: ImagingFeatureType) -> None: """ Test if the image encoder networks can be trained on segmentations from HDF5. """ logging_to_stdout() set_random_seed(0) scan_size = (6, 64, 60) dataset_contents = """subject,channel,path,label S1,week0,scan1.h5, S1,week1,scan2.h5,True S2,week0,scan3.h5, S2,week1,scan4.h5,False S3,week0,scan5.h5, S3,week1,scan6.h5,True S4,week0,scan7.h5, S4,week1,scan8.h5,True """ config = ImageEncoder(encode_channels_jointly=encode_channels_jointly, imaging_feature_type=imaging_feature_type, should_validate=False, aggregation_type=aggregation_type, scan_size=scan_size) # This fails with 16bit precision, saying "torch.nn.functional.binary_cross_entropy and torch.nn.BCELoss are # unsafe to autocast. Many models use a sigmoid layer right before the binary cross entropy layer. In this case, # combine the two layers using torch.nn.functional.binary_cross_entropy_with_logits or # torch.nn.BCEWithLogitsLoss. binary_cross_entropy_with_logits and BCEWithLogits are safe to autocast." config.use_mixed_precision = False config.set_output_to(test_output_dirs.root_dir) config.num_epochs = 1 config.local_dataset = Path() config.dataset_data_frame = pd.read_csv(StringIO(dataset_contents), sep=",", dtype=str) # Patch the load_images function that will be called once we access a dataset item image_and_seg = ImageAndSegmentations[np.ndarray]( images=np.zeros(scan_size, dtype=np.float32), segmentations=np.ones(scan_size, dtype=np.uint8)) with mock.patch('InnerEye.ML.utils.io_util.load_image_in_known_formats', return_value=image_and_seg): azure_config = get_default_azure_config() azure_config.train = True MLRunner(config, azure_config).run()
def test_download_azureml_dataset(test_output_dirs: OutputFolderForTests) -> None: dataset_name = "test-dataset" config = DummyModel() config.local_dataset = None config.azure_dataset_id = "" azure_config = get_default_azure_config() runner = MLRunner(config, azure_config=azure_config) # If the model has neither local_dataset or azure_dataset_id, mount_or_download_dataset should fail. # This mounting call must happen before any other operations on the container, because already the model # creation may need access to the dataset. with pytest.raises(ValueError) as ex: runner.setup() assert ex.value.args[0] == "The model must contain either local_dataset or azure_dataset_id." runner.project_root = test_output_dirs.root_dir # Pointing the model to a dataset folder that does not exist should raise an Exception fake_folder = runner.project_root / "foo" runner.container.local_dataset = fake_folder with pytest.raises(FileNotFoundError): runner.mount_or_download_dataset(runner.container.azure_dataset_id, runner.container.local_dataset) # If the local dataset folder exists, mount_or_download_dataset should not do anything. fake_folder.mkdir() local_dataset = runner.mount_or_download_dataset(runner.container.azure_dataset_id, runner.container.local_dataset) assert local_dataset == fake_folder # Pointing the model to a dataset in Azure should trigger a download runner.container.local_dataset = None runner.container.azure_dataset_id = dataset_name with logging_section("Starting download"): result_path = runner.mount_or_download_dataset(runner.container.azure_dataset_id, runner.container.local_dataset) # Download goes into <project_root> / "datasets" / "test_dataset" expected_path = runner.project_root / fixed_paths.DATASETS_DIR_NAME / dataset_name assert result_path == expected_path assert result_path.is_dir() dataset_csv = Path(result_path) / DATASET_CSV_FILE_NAME assert dataset_csv.is_file() # Check that each individual file in the dataset is present for folder in [1, *range(10, 20)]: sub_folder = result_path / str(folder) sub_folder.is_dir() for file in ["ct", "esophagus", "heart", "lung_l", "lung_r", "spinalcord"]: f = (sub_folder / file).with_suffix(".nii.gz") assert f.is_file()
def test_run_ml_with_segmentation_model( test_output_dirs: TestOutputDirectories) -> None: """ Test training and testing of segmentation models, when it is started together via run_ml. """ train_config = DummyModel() train_config.num_dataload_workers = 0 train_config.restrict_subjects = "1" # Increasing the test crop size should not have any effect on the results. # This is for a bug in an earlier version of the code where the wrong execution mode was used to # compute the expected mask size at training time. train_config.test_crop_size = (75, 75, 75) train_config.perform_training_set_inference = False train_config.perform_validation_and_test_set_inference = True train_config.set_output_to(test_output_dirs.root_dir) azure_config = get_default_azure_config() azure_config.train = True MLRunner(train_config, azure_config).run()
def test_dataset_consumption2() -> None: """ Creating datasets, case 2: Azure datasets, local folders and mount points given """ azure_config = get_default_azure_config() datasets = create_dataset_configs(azure_config, all_azure_dataset_ids=["1", "2"], all_dataset_mountpoints=["mp1", "mp2"], all_local_datasets=[Path("l1"), Path("l2")]) assert len(datasets) == 2 assert datasets[0].name == "1" assert datasets[1].name == "2" assert datasets[0].local_folder == Path("l1") assert datasets[1].local_folder == Path("l2") if is_linux(): # PosixPath cannot be instantiated on Windows assert datasets[0].target_folder == PosixPath("mp1") assert datasets[1].target_folder == PosixPath("mp2")
def test_image_encoder_with_segmentation( test_output_dirs: OutputFolderForTests, encode_channels_jointly: bool, aggregation_type: AggregationType, imaging_feature_type: ImagingFeatureType) -> None: """ Test if the image encoder networks can be trained on segmentations from HDF5. """ logging_to_stdout() set_random_seed(0) scan_size = (6, 64, 60) dataset_contents = """subject,channel,path,label S1,week0,scan1.h5, S1,week1,scan2.h5,True S2,week0,scan3.h5, S2,week1,scan4.h5,False S3,week0,scan5.h5, S3,week1,scan6.h5,True S4,week0,scan7.h5, S4,week1,scan8.h5,True """ config = ImageEncoder(encode_channels_jointly=encode_channels_jointly, imaging_feature_type=imaging_feature_type, should_validate=False, aggregation_type=aggregation_type, scan_size=scan_size) config.use_mixed_precision = True config.set_output_to(test_output_dirs.root_dir) config.num_epochs = 1 config.local_dataset = Path() config.dataset_data_frame = pd.read_csv(StringIO(dataset_contents), sep=",", dtype=str) # Patch the load_images function that will be called once we access a dataset item image_and_seg = ImageAndSegmentations[np.ndarray]( images=np.zeros(scan_size, dtype=np.float32), segmentations=np.ones(scan_size, dtype=np.uint8)) with mock.patch("InnerEye.ML.run_ml.is_offline_run_context", return_value=True): with mock.patch( 'InnerEye.ML.utils.io_util.load_image_in_known_formats', return_value=image_and_seg): azure_config = get_default_azure_config() azure_config.train = True MLRunner(config, azure_config=azure_config).run()
def test_run_ml_with_classification_model( test_output_dirs: TestOutputDirectories, number_of_offline_cross_validation_splits: int, number_of_cross_validation_splits_per_fold: int, model_name: str) -> None: """ Test training and testing of classification models, when it is started together via run_ml. """ logging_to_stdout() azure_config = get_default_azure_config() azure_config.train = True train_config: ScalarModelBase = ModelConfigLoader[ScalarModelBase]() \ .create_model_config_from_name(model_name) train_config.number_of_cross_validation_splits = number_of_offline_cross_validation_splits train_config.number_of_cross_validation_splits_per_fold = number_of_cross_validation_splits_per_fold train_config.set_output_to(test_output_dirs.root_dir) if train_config.perform_sub_fold_cross_validation: train_config.local_dataset = full_ml_test_data_path( "classification_data_sub_fold_cv") MLRunner(train_config, azure_config).run() _check_offline_cross_validation_output_files(train_config) if train_config.is_regression_model: assert (train_config.outputs_folder / "0" / "error_plot_4.png").is_file() if train_config.perform_cross_validation: # Test that the result files can be correctly picked up by the cross validation routine. # For that, we point the downloader to the local results folder. The core download method # recognizes run_recovery_id == None as the signal to read from the local_run_results folder. config_and_files = get_config_and_results_for_offline_runs( train_config) result_files = config_and_files.files # One file for VAL and one for TRAIN for each child run assert len( result_files ) == train_config.get_total_number_of_cross_validation_runs() * 2 for file in result_files: assert file.execution_mode != ModelExecutionMode.TEST assert file.dataset_csv_file is not None assert file.dataset_csv_file.exists() assert file.metrics_file is not None assert file.metrics_file.exists()
def test_download_azureml_dataset( test_output_dirs: OutputFolderForTests) -> None: dataset_name = "test-dataset" config = ModelConfigBase(should_validate=False) azure_config = get_default_azure_config() runner = MLRunner(config, azure_config) runner.project_root = test_output_dirs.root_dir # If the model has neither local_dataset or azure_dataset_id, mount_or_download_dataset should fail. with pytest.raises(ValueError): runner.mount_or_download_dataset() # Pointing the model to a dataset folder that does not exist should raise an Exception fake_folder = runner.project_root / "foo" runner.model_config.local_dataset = fake_folder with pytest.raises(FileNotFoundError): runner.mount_or_download_dataset() # If the local dataset folder exists, mount_or_download_dataset should not do anything. fake_folder.mkdir() local_dataset = runner.mount_or_download_dataset() assert local_dataset == fake_folder # Pointing the model to a dataset in Azure should trigger a download runner.model_config.local_dataset = None runner.model_config.azure_dataset_id = dataset_name with logging_section("Starting download"): result_path = runner.mount_or_download_dataset() # Download goes into <project_root> / "datasets" / "test_dataset" expected_path = runner.project_root / fixed_paths.DATASETS_DIR_NAME / dataset_name assert result_path == expected_path assert result_path.is_dir() dataset_csv = Path(result_path) / DATASET_CSV_FILE_NAME assert dataset_csv.is_file() # Check that each individual file in the dataset is present for folder in [1, *range(10, 20)]: sub_folder = result_path / str(folder) sub_folder.is_dir() for file in [ "ct", "esophagus", "heart", "lung_l", "lung_r", "spinalcord" ]: f = (sub_folder / file).with_suffix(".nii.gz") assert f.is_file()
def test_register_model_invalid() -> None: ws = get_default_workspace() config = get_model_loader().create_model_config_from_name("Lung") with pytest.raises(Exception): ml_runner = MLRunner(config, None) ml_runner.register_segmentation_model( run=Run.get_context(), workspace=ws, best_epoch=0, best_epoch_dice=0, checkpoint_paths=checkpoint_paths, model_proc=ModelProcessing.DEFAULT) with pytest.raises(Exception): ml_runner = MLRunner(config, get_default_azure_config()) ml_runner.register_segmentation_model( best_epoch=0, best_epoch_dice=0, checkpoint_paths=checkpoint_paths, model_proc=ModelProcessing.DEFAULT)
def test_runner_restart(test_output_dirs: OutputFolderForTests) -> None: """ Test if starting training from a folder where the checkpoints folder already has recovery checkpoints picks up that it is a recovery run. Also checks that we update the start epoch in the config at loading time. """ model_config = DummyClassification() model_config.set_output_to(test_output_dirs.root_dir) model_config.num_epochs = FIXED_EPOCH + 2 # We save all checkpoints - if recovery works as expected we should have a new checkpoint for epoch 4, 5. model_config.recovery_checkpoint_save_interval = 1 model_config.recovery_checkpoints_save_last_k = -1 runner = MLRunner(model_config=model_config) runner.setup(use_mount_or_download_dataset=False) # Epochs are 0 based for saving create_model_and_store_checkpoint(model_config, runner.container.checkpoint_folder / f"{RECOVERY_CHECKPOINT_FILE_NAME}_epoch=" f"{FIXED_EPOCH - 1}{CHECKPOINT_SUFFIX}", weights_only=False) azure_config = get_default_azure_config() checkpoint_handler = CheckpointHandler( azure_config=azure_config, container=runner.container, project_root=test_output_dirs.root_dir) _, storing_logger = model_train(checkpoint_handler=checkpoint_handler, container=runner.container) # We expect to have 4 checkpoints, FIXED_EPOCH (recovery), FIXED_EPOCH+1, FIXED_EPOCH and best. assert len(os.listdir(runner.container.checkpoint_folder)) == 4 assert (runner.container.checkpoint_folder / f"{RECOVERY_CHECKPOINT_FILE_NAME}_epoch=" f"{FIXED_EPOCH - 1}{CHECKPOINT_SUFFIX}").exists() assert (runner.container.checkpoint_folder / f"{RECOVERY_CHECKPOINT_FILE_NAME}_epoch=" f"{FIXED_EPOCH}{CHECKPOINT_SUFFIX}").exists() assert (runner.container.checkpoint_folder / f"{RECOVERY_CHECKPOINT_FILE_NAME}_epoch=" f"{FIXED_EPOCH + 1}{CHECKPOINT_SUFFIX}").exists() assert (runner.container.checkpoint_folder / BEST_CHECKPOINT_FILE_NAME_WITH_SUFFIX).exists() # Check that we really restarted epoch from epoch FIXED_EPOCH. assert list(storing_logger.epochs) == [FIXED_EPOCH, FIXED_EPOCH + 1] # type: ignore
def test_run_ml_with_multi_label_sequence_in_crossval(test_output_dirs: TestOutputDirectories) -> None: """ Test training and testing of sequence models that predicts at multiple time points, including aggregation of cross validation results. """ logging_to_stdout() config = ToyMultiLabelSequenceModel(should_validate=False) assert config.get_target_indices() == [1, 2, 3] expected_prediction_targets = ["Seq_pos 01", "Seq_pos 02", "Seq_pos 03"] target_indices = config.get_target_indices() assert target_indices assert len(target_indices) == len(expected_prediction_targets) config.set_output_to(test_output_dirs.root_dir) config.dataset_data_frame = _get_multi_label_sequence_dataframe() config.pre_process_dataset_dataframe() config.num_epochs = 1 config.number_of_cross_validation_splits = 2 azure_config = get_default_azure_config() azure_config.train = True MLRunner(config, azure_config).run()
def test_model_inference_on_single_run(test_output_dirs: OutputFolderForTests) -> None: falllback_run_id = FALLBACK_HELLO_CONTAINER_RUN files_to_check = ["test_mse.txt", "test_mae.txt"] training_run = get_most_recent_run(fallback_run_id_for_local_execution=falllback_run_id) all_training_files = training_run.get_file_names() for file in files_to_check: assert f"outputs/{file}" in all_training_files, f"{file} is missing" training_folder = test_output_dirs.root_dir / "training" training_folder.mkdir() training_files = [training_folder / file for file in files_to_check] for file, download_path in zip(files_to_check, training_files): training_run.download_file(f"outputs/{file}", output_file_path=str(download_path)) container = HelloContainer() container.set_output_to(test_output_dirs.root_dir) container.model_id = get_most_recent_model_id(fallback_run_id_for_local_execution=falllback_run_id) azure_config = get_default_azure_config() azure_config.train = False ml_runner = MLRunner(container=container, azure_config=azure_config, project_root=test_output_dirs.root_dir) ml_runner.setup() ml_runner.run() inference_files = [container.outputs_folder / file for file in files_to_check] for inference_file in inference_files: assert inference_file.exists(), f"{inference_file} is missing" for training_file, inference_file in zip(training_files, inference_files): training_lines = training_file.read_text().splitlines() inference_lines = inference_file.read_text().splitlines() # We expect all the files we are reading to have a single float value assert len(training_lines) == 1 train_value = float(training_lines[0].strip()) assert len(inference_lines) == 1 inference_value = float(inference_lines[0].strip()) assert inference_value == pytest.approx(train_value, 1e-6)
def test_register_and_score_model(is_ensemble: bool, dataset_expected_spacing_xyz: Any, model_outside_package: bool, test_output_dirs: OutputFolderForTests) -> None: """ End-to-end test which ensures the scoring pipeline is functioning as expected by performing the following: 1) Registering a pre-trained model to AML 2) Checking that a model zip from the registered model can be created successfully 3) Calling the scoring pipeline to check inference can be run from the published model successfully """ ws = get_default_workspace() # Get an existing config as template loader = get_model_loader("Tests.ML.configs" if model_outside_package else None) config: SegmentationModelBase = loader.create_model_config_from_name( model_name="BasicModel2EpochsOutsidePackage" if model_outside_package else "BasicModel2Epochs" ) config.dataset_expected_spacing_xyz = dataset_expected_spacing_xyz config.set_output_to(test_output_dirs.root_dir) # copy checkpoints into the outputs (simulating a run) stored_checkpoints = full_ml_test_data_path(os.path.join("train_and_test_data", "checkpoints")) shutil.copytree(str(stored_checkpoints), str(config.checkpoint_folder)) paths = [config.checkpoint_folder / "1_checkpoint.pth.tar"] checkpoints = paths * 2 if is_ensemble else paths model = None model_path = None # Mocking to get the source from the current directory # the score.py and python_wrapper.py cannot be moved inside the InnerEye package, which will be the # only code running (if these tests are run on the package). with mock.patch('InnerEye.Common.fixed_paths.repository_root_directory', return_value=tests_root_directory().parent): try: tags = {"model_name": config.model_name} azure_config = get_default_azure_config() if model_outside_package: azure_config.extra_code_directory = "Tests" # contains DummyModel deployment_hook = lambda cfg, azure_cfg, mdl, is_ens: (Path(cfg.model_name), azure_cfg.docker_shm_size) ml_runner = MLRunner(config, azure_config, model_deployment_hook=deployment_hook) model, deployment_path, deployment_details = ml_runner.register_segmentation_model( workspace=ws, tags=tags, best_epoch=0, best_epoch_dice=0, checkpoint_paths=checkpoints, model_proc=ModelProcessing.DEFAULT) assert model is not None model_path = Path(model.get_model_path(model.name, model.version, ws)) assert (model_path / fixed_paths.ENVIRONMENT_YAML_FILE_NAME).exists() assert (model_path / Path("InnerEye/ML/runner.py")).exists() assert deployment_path == Path(config.model_name) assert deployment_details == azure_config.docker_shm_size # move test data into the data folder to simulate an actual run train_and_test_data_dir = full_ml_test_data_path("train_and_test_data") img_channel_1_name = "id1_channel1.nii.gz" img_channel_1_path = train_and_test_data_dir / img_channel_1_name img_channel_2_name = "id1_channel2.nii.gz" img_channel_2_path = train_and_test_data_dir / img_channel_2_name # download the registered model and test that we can run the score pipeline on it model_root = Path(model.download(str(test_output_dirs.root_dir))) # create a dummy datastore to store model checkpoints and image data # this simulates the code shapshot being executed in a real run test_datastore = test_output_dirs.root_dir / "test_datastore" shutil.move( str(model_root / "test_outputs"), str(test_datastore / RELATIVE_TEST_OUTPUTS_PATH) ) data_root = test_datastore / DEFAULT_DATA_FOLDER os.makedirs(data_root) shutil.copy(str(img_channel_1_path), data_root) shutil.copy(str(img_channel_2_path), data_root) # run score pipeline as a separate process using the python_wrapper.py code to simulate a real run return_code = SubprocessConfig(process="python", args=[ str(model_root / "python_wrapper.py"), "--spawnprocess=python", str(model_root / "score.py"), f"--data-folder={str(test_datastore)}", f"--test_image_channels={img_channel_1_name},{img_channel_2_name}", "--use_gpu=False" ]).spawn_and_monitor_subprocess() # check that the process completed as expected assert return_code == 0 expected_segmentation_path = Path(model_root) / DEFAULT_RESULT_IMAGE_NAME assert expected_segmentation_path.exists() # sanity check the resulting segmentation expected_shape = get_nifti_shape(img_channel_1_path) image_header = get_unit_image_header() assert_nifti_content(str(expected_segmentation_path), expected_shape, image_header, [0], np.ubyte) finally: # delete the registered model, and any downloaded artifacts shutil.rmtree(test_output_dirs.root_dir) if model and model_path: model.delete() shutil.rmtree(model_path)