def test_copy_child_paths_to_folder(is_ensemble: bool, extra_code_directory: str, test_output_dirs: OutputFolderForTests) -> None: azure_config = AzureConfig(extra_code_directory=extra_code_directory) fake_model = SegmentationModelBase(should_validate=False) fake_model.set_output_to(test_output_dirs.root_dir) # To simulate ensemble models, there are two checkpoints, one in the root dir and one in a folder checkpoints_absolute, checkpoints_relative = create_checkpoints(fake_model, is_ensemble) # Simulate a project root: We can't derive that from the repository root because that might point # into Python's package folder project_root = Path(__file__).parent.parent ml_runner = MLRunner(model_config=fake_model, azure_config=azure_config, project_root=project_root) model_folder = test_output_dirs.root_dir / "final" ml_runner.copy_child_paths_to_folder(model_folder=model_folder, checkpoint_paths=checkpoints_absolute) expected_files = [ fixed_paths.ENVIRONMENT_YAML_FILE_NAME, fixed_paths.MODEL_INFERENCE_JSON_FILE_NAME, "InnerEye/ML/runner.py", "InnerEye/ML/model_testing.py", "InnerEye/Common/fixed_paths.py", "InnerEye/Common/common_util.py", ] for r in checkpoints_relative: expected_files.append(f"{CHECKPOINT_FOLDER}/{r}") for expected_file in expected_files: assert (model_folder / expected_file).is_file(), f"File missing: {expected_file}" trm = model_folder / "TestsOutsidePackage/test_register_model.py" if extra_code_directory: assert trm.is_file() else: assert not trm.is_file()
def test_score_image_dicom_mock_none( test_output_dirs: OutputFolderForTests) -> None: """ Test that dicom in and dicom-rt out works. Finally there is no mocking and full image scoring is run using the PassThroughModel. :param test_output_dirs: Test output directories. """ model_config = PassThroughModel() model_config.set_output_to(test_output_dirs.root_dir) checkpoint_path = model_config.checkpoint_folder / "checkpoint.ckpt" create_model_and_store_checkpoint(model_config, checkpoint_path) azure_config = AzureConfig() project_root = Path(__file__).parent.parent ml_runner = MLRunner(model_config=model_config, azure_config=azure_config, project_root=project_root) model_folder = test_output_dirs.root_dir / "final" ml_runner.copy_child_paths_to_folder(model_folder=model_folder, checkpoint_paths=[checkpoint_path]) zipped_dicom_series_path = zip_dicom_series(model_folder) score_pipeline_config = ScorePipelineConfig( data_folder=zipped_dicom_series_path.parent, model_folder=str(model_folder), image_files=[str(zipped_dicom_series_path)], result_image_name=HNSEGMENTATION_FILE.name, use_gpu=False, use_dicom=True) segmentation = score_image(score_pipeline_config) assert_zip_file_contents(segmentation, HN_DICOM_RT_ZIPPED, model_folder)
def model_train_unittest(config: Optional[DeepLearningConfig], dirs: OutputFolderForTests, checkpoint_handler: Optional[CheckpointHandler] = None, lightning_container: Optional[LightningContainer] = None) -> \ Tuple[StoringLogger, CheckpointHandler]: """ A shortcut for running model training in the unit test suite. It runs training for the given config, with the default checkpoint handler initialized to point to the test output folder specified in dirs. :param config: The configuration of the model to train. :param dirs: The test fixture that provides an output folder for the test. :param lightning_container: An optional LightningContainer object that will be pass through to the training routine. :param checkpoint_handler: The checkpoint handler that should be used for training. If not provided, it will be created via get_default_checkpoint_handler. :return: Tuple[StoringLogger, CheckpointHandler] """ runner = MLRunner(model_config=config, container=lightning_container) # Setup will set random seeds before model creation, and set the model in the container. # It will also set random seeds correctly. Later we use so initialized container. # For all tests running in AzureML, we need to skip the downloading of datasets that would otherwise happen, # because all unit test configs come with their own local dataset already. runner.setup(use_mount_or_download_dataset=False) if checkpoint_handler is None: azure_config = get_default_azure_config() checkpoint_handler = CheckpointHandler(azure_config=azure_config, container=runner.container, project_root=dirs.root_dir) _, storing_logger = model_train(checkpoint_handler=checkpoint_handler, container=runner.container) return storing_logger, checkpoint_handler # type: ignore
def test_model_name_is_set(test_output_dirs: OutputFolderForTests) -> None: container = DummyContainerWithModel() container.local_dataset = test_output_dirs.root_dir runner = MLRunner(model_config=None, container=container) runner.setup() expected_name = "DummyContainerWithModel" assert runner.container._model_name == expected_name assert expected_name in str(runner.container.outputs_folder)
def _create_container( extra_local_dataset_paths: List[Path] = [], extra_azure_dataset_ids: List[str] = []) -> LightningContainer: container = DummyContainerWithModel() container.local_dataset = test_output_dirs.root_dir container.extra_local_dataset_paths = extra_local_dataset_paths # type: ignore container.extra_azure_dataset_ids = extra_azure_dataset_ids runner = MLRunner(model_config=None, container=container) runner.setup() return runner.container
def test_optim_params1(test_output_dirs: OutputFolderForTests) -> None: """ Test if the optimizer parameters are read correctly for InnerEye configs. """ model = DummyModel() model.set_output_to(test_output_dirs.root_dir) runner = MLRunner(model_config=model) runner.setup() lightning_model = runner.container.model optim, _ = lightning_model.configure_optimizers() assert optim[0].param_groups[0]["lr"] == 1e-3
def test_optim_params2(test_output_dirs: OutputFolderForTests) -> None: """ Test if the optimizer parameters are read correctly for containers. """ container = DummyContainerWithModel() container.local_dataset = test_output_dirs.root_dir runner = MLRunner(model_config=None, container=container) runner.setup() lightning_model = runner.container.model optim, _ = lightning_model.configure_optimizers() expected_lr = 1e-1 assert container.l_rate == expected_lr assert optim[0].param_groups[0]["lr"] == expected_lr
def test_score_image_dicom_mock_run_store( test_output_dirs: OutputFolderForTests) -> None: """ Test that dicom in and dicom-rt out works, by mocking out run and store functions. This mocks out run_inference and store_as_ubyte_nifti so that init_from_model_inference_json is tested in addition to the tests in test_score_image_dicom_mock_all. :param test_output_dirs: Test output directories. """ mock_segmentation = {'mock_segmentation': True} model_config = DummyModel() model_config.set_output_to(test_output_dirs.root_dir) checkpoint_path = model_config.checkpoint_folder / "checkpoint.ckpt" create_model_and_store_checkpoint(model_config, checkpoint_path) azure_config = AzureConfig() project_root = Path(__file__).parent.parent ml_runner = MLRunner(model_config=model_config, azure_config=azure_config, project_root=project_root) model_folder = test_output_dirs.root_dir / "final" ml_runner.copy_child_paths_to_folder(model_folder=model_folder, checkpoint_paths=[checkpoint_path]) zipped_dicom_series_path = test_output_dirs.root_dir / "temp_pack_dicom_series" / "dicom_series.zip" zip_known_dicom_series(zipped_dicom_series_path) score_pipeline_config = ScorePipelineConfig( data_folder=zipped_dicom_series_path.parent, model_folder=str(model_folder), image_files=[str(zipped_dicom_series_path)], result_image_name=HNSEGMENTATION_FILE.name, use_gpu=False, use_dicom=True, model_id="Dummy:1") with mock.patch('score.run_inference', return_value=mock_segmentation) as mock_run_inference: with mock.patch( 'score.store_as_ubyte_nifti', return_value=HNSEGMENTATION_FILE) as mock_store_as_ubyte_nifti: segmentation = score_image(score_pipeline_config) assert_zip_file_contents(segmentation, HN_DICOM_RT_ZIPPED, model_folder) mock_run_inference.assert_called() mock_store_as_ubyte_nifti.assert_called()
def test_run_ml_with_sequence_model(use_combined_model: bool, imaging_feature_type: ImagingFeatureType, test_output_dirs: TestOutputDirectories) -> None: """ Test training and testing of sequence models, when it is started together via run_ml. """ logging_to_stdout() config = ToySequenceModel(use_combined_model, imaging_feature_type, should_validate=False, sequence_target_positions=[2, 10]) config.set_output_to(test_output_dirs.root_dir) config.dataset_data_frame = _get_mock_sequence_dataset() config.num_epochs = 1 config.max_batch_grad_cam = 1 # make sure we are testing with at least one sequence position that will not exist # to ensure correct handling of sequences that do not contain all the expected target positions assert max(config.sequence_target_positions) > config.dataset_data_frame[config.sequence_column].astype(float).max() # Patch the load_images function that will be called once we access a dataset item image_and_seg = ImageAndSegmentations[np.ndarray](images=np.random.uniform(0, 1, SCAN_SIZE), segmentations=np.random.randint(0, 2, SCAN_SIZE)) with mock.patch('InnerEye.ML.utils.io_util.load_image_in_known_formats', return_value=image_and_seg): azure_config = get_default_azure_config() azure_config.train = True MLRunner(config, azure_config).run()
def test_run_ml_with_classification_model( test_output_dirs: OutputFolderForTests, number_of_offline_cross_validation_splits: int, model_name: str) -> None: """ Test training and testing of classification models, when it is started together via run_ml. """ logging_to_stdout() azure_config = get_default_azure_config() azure_config.train = True config: ScalarModelBase = ModelConfigLoader[ScalarModelBase]() \ .create_model_config_from_name(model_name) config.number_of_cross_validation_splits = number_of_offline_cross_validation_splits config.set_output_to(test_output_dirs.root_dir) # Trying to run DDP from the test suite hangs, hence restrict to single GPU. config.max_num_gpus = 1 MLRunner(config, azure_config).run() _check_offline_cross_validation_output_files(config) if config.perform_cross_validation: # Test that the result files can be correctly picked up by the cross validation routine. # For that, we point the downloader to the local results folder. The core download method # recognizes run_recovery_id == None as the signal to read from the local_run_results folder. config_and_files = get_config_and_results_for_offline_runs(config) result_files = config_and_files.files # One file for VAL and one for TRAIN for each child run assert len(result_files ) == config.get_total_number_of_cross_validation_runs() * 2 for file in result_files: assert file.execution_mode != ModelExecutionMode.TEST assert file.dataset_csv_file is not None assert file.dataset_csv_file.exists() assert file.metrics_file is not None assert file.metrics_file.exists()
def test_register_model_skip() -> None: """ If the AzureML workspace can't be read, model registration should be skipped and return None. """ checkpoint_paths = [ full_ml_test_data_path('checkpoints') / '1_checkpoint.pth.tar' ] config = get_model_loader().create_model_config_from_name("Lung") ml_runner = MLRunner(config, None) raises = mock.Mock() raises.side_effect = Exception with mock.patch.object(AzureConfig, 'get_workspace', raises): model, deployment_result = ml_runner.register_segmentation_model( model_description="", checkpoint_paths=checkpoint_paths, model_proc=ModelProcessing.DEFAULT) assert model is None assert deployment_result is None
def test_score_image_dicom_mock_run( test_output_dirs: OutputFolderForTests) -> None: """ Test that dicom in and dicom-rt out works, by mocking out only the run scoring function. This mocks out run_inference so that store_as_ubyte_nifti is tested in addition to the tests in test_score_image_dicom_mock_run_store. :param test_output_dirs: Test output directories. """ model_config = DummyModel() model_config.set_output_to(test_output_dirs.root_dir) checkpoint_path = model_config.checkpoint_folder / "checkpoint.ckpt" create_model_and_store_checkpoint(model_config, checkpoint_path) azure_config = AzureConfig() project_root = Path(__file__).parent.parent ml_runner = MLRunner(model_config=model_config, azure_config=azure_config, project_root=project_root) model_folder = test_output_dirs.root_dir / "final" ml_runner.copy_child_paths_to_folder(model_folder=model_folder, checkpoint_paths=[checkpoint_path]) zipped_dicom_series_path = zip_dicom_series(model_folder) score_pipeline_config = ScorePipelineConfig( data_folder=zipped_dicom_series_path.parent, model_folder=str(model_folder), image_files=[str(zipped_dicom_series_path)], result_image_name=HNSEGMENTATION_FILE.name, use_gpu=False, use_dicom=True) image_with_header = io_util.load_nifti_image(HNSEGMENTATION_FILE) with mock.patch( 'score.run_inference', return_value=image_with_header.image) as mock_run_inference: segmentation = score_image(score_pipeline_config) assert_zip_file_contents(segmentation, HN_DICOM_RT_ZIPPED, model_folder) mock_run_inference.assert_called()
def create_ml_runner(self) -> Any: """ Create and return an ML runner using the attributes of this Runner object. """ # This import statement cannot be at the beginning of the file because it will cause import # of packages that are not available inside the azure_runner.yml environment: torch, blobxfer. # That is also why we specify the return type as Any rather than MLRunner. from InnerEye.ML.run_ml import MLRunner return MLRunner(self.model_config, self.azure_config, self.project_root, self.model_deployment_hook)
def test_get_child_paths(is_ensemble: bool, extra_code_directory: str) -> None: checkpoints = checkpoint_paths * 2 if is_ensemble else checkpoint_paths path_to_root = tests_root_directory().parent azure_config = AzureConfig(extra_code_directory=extra_code_directory) fake_model = ModelConfigBase(azure_dataset_id="fake_dataset_id") ml_runner = MLRunner(model_config=fake_model, azure_config=azure_config, project_root=path_to_root) child_paths = ml_runner.get_child_paths(checkpoints) assert fixed_paths.ENVIRONMENT_YAML_FILE_NAME in child_paths assert fixed_paths.MODEL_INFERENCE_JSON_FILE_NAME in child_paths assert str(Path("InnerEye/ML/runner.py")) in child_paths assert str(Path("InnerEye/ML/model_testing.py")) in child_paths assert str(Path("InnerEye/Common/fixed_paths.py")) in child_paths assert str(Path("InnerEye/Common/common_util.py")) in child_paths trm = str(Path("TestsOutsidePackage/test_register_model.py")) if extra_code_directory: assert trm in child_paths else: assert trm not in child_paths assert all([x.relative_to(path_to_root) for x in checkpoints])
def main(yaml_file_path: Path) -> None: """ Invoke either by * specifying a model, '--model Lung' * or specifying dataset and normalization parameters separately: --azure_dataset_id=foo --norm_method=None In addition, the arguments '--image_channel' and '--gt_channel' must be specified (see below). """ config, runner_config, args = get_configs( SegmentationModelBase(should_validate=False), yaml_file_path) local_dataset = MLRunner( config, azure_config=runner_config).mount_or_download_dataset( config.azure_dataset_id, config.local_dataset) assert local_dataset is not None dataframe = pd.read_csv(local_dataset / DATASET_CSV_FILE_NAME) normalizer_config = NormalizeAndVisualizeConfig(**args) actual_mask_channel = None if normalizer_config.ignore_mask else config.mask_id image_channel = normalizer_config.image_channel or config.image_channels[0] if not image_channel: raise ValueError( "No image channel selected. Specify a model by name, or use the image_channel argument." ) gt_channel = normalizer_config.gt_channel or config.ground_truth_ids[0] if not gt_channel: raise ValueError( "No GT channel selected. Specify a model by name, or use the gt_channel argument." ) dataset_sources = load_dataset_sources( dataframe, local_dataset_root_folder=local_dataset, image_channels=[image_channel], ground_truth_channels=[gt_channel], mask_channel=actual_mask_channel) result_folder = local_dataset if normalizer_config.result_folder is not None: result_folder = result_folder / normalizer_config.result_folder if not result_folder.is_dir(): result_folder.mkdir() all_patient_ids = [*dataset_sources.keys()] if normalizer_config.only_first == 0: patient_ids_to_process = all_patient_ids else: patient_ids_to_process = all_patient_ids[:normalizer_config.only_first] args_file = result_folder / ARGS_TXT args_file.write_text(" ".join(sys.argv[1:])) config_file = result_folder / "config.txt" config_file.write_text(str(config)) normalizer = PhotometricNormalization(config) for patient_id in patient_ids_to_process: print(f"Starting to process patient {patient_id}") images = load_images_from_dataset_source(dataset_sources[patient_id]) plotting.plot_normalization_result(images, normalizer, result_folder, result_prefix=image_channel)
def create_ml_runner(self) -> MLRunner: """ Create and return an ML runner using the attributes of this Runner object. """ return MLRunner( model_config=self.model_config, container=self.lightning_container, azure_config=self.azure_config, project_root=self.project_root, post_cross_validation_hook=self.post_cross_validation_hook, model_deployment_hook=self.model_deployment_hook)
def test_download_azureml_dataset(test_output_dirs: OutputFolderForTests) -> None: dataset_name = "test-dataset" config = DummyModel() config.local_dataset = None config.azure_dataset_id = "" azure_config = get_default_azure_config() runner = MLRunner(config, azure_config=azure_config) # If the model has neither local_dataset or azure_dataset_id, mount_or_download_dataset should fail. # This mounting call must happen before any other operations on the container, because already the model # creation may need access to the dataset. with pytest.raises(ValueError) as ex: runner.setup() assert ex.value.args[0] == "The model must contain either local_dataset or azure_dataset_id." runner.project_root = test_output_dirs.root_dir # Pointing the model to a dataset folder that does not exist should raise an Exception fake_folder = runner.project_root / "foo" runner.container.local_dataset = fake_folder with pytest.raises(FileNotFoundError): runner.mount_or_download_dataset(runner.container.azure_dataset_id, runner.container.local_dataset) # If the local dataset folder exists, mount_or_download_dataset should not do anything. fake_folder.mkdir() local_dataset = runner.mount_or_download_dataset(runner.container.azure_dataset_id, runner.container.local_dataset) assert local_dataset == fake_folder # Pointing the model to a dataset in Azure should trigger a download runner.container.local_dataset = None runner.container.azure_dataset_id = dataset_name with logging_section("Starting download"): result_path = runner.mount_or_download_dataset(runner.container.azure_dataset_id, runner.container.local_dataset) # Download goes into <project_root> / "datasets" / "test_dataset" expected_path = runner.project_root / fixed_paths.DATASETS_DIR_NAME / dataset_name assert result_path == expected_path assert result_path.is_dir() dataset_csv = Path(result_path) / DATASET_CSV_FILE_NAME assert dataset_csv.is_file() # Check that each individual file in the dataset is present for folder in [1, *range(10, 20)]: sub_folder = result_path / str(folder) sub_folder.is_dir() for file in ["ct", "esophagus", "heart", "lung_l", "lung_r", "spinalcord"]: f = (sub_folder / file).with_suffix(".nii.gz") assert f.is_file()
def test_non_image_encoder( test_output_dirs: OutputFolderForTests, hidden_layer_num_feature_channels: Optional[int]) -> None: """ Test if we can build a simple MLP model that only feeds off non-image features. """ dataset_folder = Path(test_output_dirs.make_sub_dir("dataset")) dataset_contents = _get_fake_dataset_contents() (dataset_folder / DATASET_CSV_FILE_NAME).write_text(dataset_contents) config = NonImageEncoder( should_validate=False, hidden_layer_num_feature_channels=hidden_layer_num_feature_channels) config.local_dataset = dataset_folder config.set_output_to(test_output_dirs.root_dir) config.max_batch_grad_cam = 1 config.validate() # run model training _, checkpoint_handler = model_train_unittest( config, output_folder=test_output_dirs) # run model inference runner = MLRunner(config) runner.setup() runner.model_inference_train_and_test( checkpoint_paths=checkpoint_handler.get_checkpoints_to_test()) assert config.get_total_number_of_non_imaging_features() == 18
def test_download_azureml_dataset( test_output_dirs: OutputFolderForTests) -> None: dataset_name = "test-dataset" config = ModelConfigBase(should_validate=False) azure_config = get_default_azure_config() runner = MLRunner(config, azure_config) runner.project_root = test_output_dirs.root_dir # If the model has neither local_dataset or azure_dataset_id, mount_or_download_dataset should fail. with pytest.raises(ValueError): runner.mount_or_download_dataset() # Pointing the model to a dataset folder that does not exist should raise an Exception fake_folder = runner.project_root / "foo" runner.model_config.local_dataset = fake_folder with pytest.raises(FileNotFoundError): runner.mount_or_download_dataset() # If the local dataset folder exists, mount_or_download_dataset should not do anything. fake_folder.mkdir() local_dataset = runner.mount_or_download_dataset() assert local_dataset == fake_folder # Pointing the model to a dataset in Azure should trigger a download runner.model_config.local_dataset = None runner.model_config.azure_dataset_id = dataset_name with logging_section("Starting download"): result_path = runner.mount_or_download_dataset() # Download goes into <project_root> / "datasets" / "test_dataset" expected_path = runner.project_root / fixed_paths.DATASETS_DIR_NAME / dataset_name assert result_path == expected_path assert result_path.is_dir() dataset_csv = Path(result_path) / DATASET_CSV_FILE_NAME assert dataset_csv.is_file() # Check that each individual file in the dataset is present for folder in [1, *range(10, 20)]: sub_folder = result_path / str(folder) sub_folder.is_dir() for file in [ "ct", "esophagus", "heart", "lung_l", "lung_r", "spinalcord" ]: f = (sub_folder / file).with_suffix(".nii.gz") assert f.is_file()
def _test_mount_for_lightning_container(test_output_dirs: OutputFolderForTests, is_offline_run: bool, local_dataset: Optional[Path], azure_dataset: str, is_lightning_model: bool) -> LightningContainer: config: Optional[DeepLearningConfig] = None container: Optional[LightningContainer] = None if is_lightning_model: container = DummyContainerWithDatasets() container.azure_dataset_id = azure_dataset container.local_dataset = local_dataset else: config = DummyModel() config.azure_dataset_id = azure_dataset config.local_dataset = local_dataset # The legacy InnerEye models require an existing dataset_csv file present in the dataset folder. Create that. download_path = test_output_dirs.root_dir / "downloaded" mount_path = test_output_dirs.root_dir / "mounted" if not is_lightning_model: train_and_test_data = "train_and_test_data" for path in [download_path, mount_path, test_output_dirs.root_dir]: # If destination folder exists, delete content to ensure consistency and avoid 'FileExistsError' if (path / train_and_test_data).is_dir(): shutil.rmtree(path / train_and_test_data) # Creates directory structure and copy data shutil.copytree(full_ml_test_data_path(train_and_test_data), path / train_and_test_data) # Copy 'data.csv' file shutil.copy(full_ml_test_data_path(DATASET_CSV_FILE_NAME), path / DATASET_CSV_FILE_NAME) with mock.patch("InnerEye.ML.run_ml.MLRunner.is_offline_run", is_offline_run): with mock.patch("InnerEye.ML.run_ml.download_dataset", return_value=download_path): with mock.patch("InnerEye.ML.run_ml.try_to_mount_input_dataset", return_value=mount_path): runner = MLRunner(config, container=container, azure_config=None, project_root=test_output_dirs.root_dir) runner.setup() return runner.container
def test_run_ml_with_multi_label_sequence_model( test_output_dirs: OutputFolderForTests) -> None: """ Test training and testing of sequence models that predicts at multiple time points, when it is started via run_ml. """ logging_to_stdout() config = ToyMultiLabelSequenceModel(should_validate=False) assert config.get_target_indices() == [1, 2, 3] expected_prediction_targets = [ f"{SEQUENCE_POSITION_HUE_NAME_PREFIX} {x}" for x in ["01", "02", "03"] ] _target_indices = config.get_target_indices() assert _target_indices is not None assert len(_target_indices) == len(expected_prediction_targets) metrics_dict = create_metrics_dict_for_scalar_models(config) assert metrics_dict.get_hue_names( include_default=False) == expected_prediction_targets config.set_output_to(test_output_dirs.root_dir) # Create a fake dataset directory to make config validation pass config.local_dataset = test_output_dirs.root_dir config.dataset_data_frame = _get_multi_label_sequence_dataframe() config.pre_process_dataset_dataframe() config.num_epochs = 1 config.max_batch_grad_cam = 1 azure_config = get_default_azure_config() azure_config.train = True MLRunner(config, azure_config).run() # The metrics file should have one entry per epoch per subject per prediction target, # for all the 3 prediction targets. metrics_file = config.outputs_folder / "Train" / SUBJECT_METRICS_FILE_NAME assert metrics_file.exists() metrics = pd.read_csv(metrics_file) assert LoggingColumns.Patient.value in metrics assert LoggingColumns.Epoch.value in metrics assert LoggingColumns.Hue.value in metrics assert metrics[LoggingColumns.Hue.value].unique().tolist( ) == expected_prediction_targets group_by_subject = metrics.groupby( by=[LoggingColumns.Patient.value, LoggingColumns.Epoch.value]) expected_prediction_target_lengths = [3, 2, 3, 3] for i, x in enumerate(group_by_subject): assert len(x[1]) == expected_prediction_target_lengths[i] group_by_subject_and_target = metrics.groupby(by=[ LoggingColumns.Patient.value, LoggingColumns.Epoch.value, LoggingColumns.Hue.value ]) for _, group in group_by_subject_and_target: assert len(group) == 1
def test_file_system_with_subfolders( test_output_dirs: OutputFolderForTests) -> None: """ Test if a subfolder can be created within the output folder structure, for use with cross validation. """ model = DummyModel() model.set_output_to(test_output_dirs.root_dir) container = InnerEyeContainer(model) # File system should be copied from model config to container assert container.file_system_config == model.file_system_config runner = MLRunner(model_config=model) runner.setup() assert str(runner.container.outputs_folder).endswith(model.model_name) output_subfolder = "foo" expected_folder = runner.container.outputs_folder / output_subfolder runner = MLRunner(model_config=model, output_subfolder=output_subfolder) runner.setup() assert runner.container.outputs_folder == expected_folder
def test_regression_test(test_output_dirs: OutputFolderForTests) -> None: """ Test that the file comparison for regression tests is actually called in the workflow. """ container = DummyContainerWithModel() container.local_dataset = test_output_dirs.root_dir container.regression_test_folder = Path(str(uuid.uuid4().hex)) runner = MLRunner(container=container) runner.setup() with pytest.raises(ValueError) as ex: runner.run() assert "Folder with expected files does not exist" in str(ex)
def test_register_model_invalid() -> None: ws = get_default_workspace() config = get_model_loader().create_model_config_from_name("Lung") with pytest.raises(Exception): ml_runner = MLRunner(config, None) ml_runner.register_segmentation_model( run=Run.get_context(), workspace=ws, best_epoch=0, best_epoch_dice=0, checkpoint_paths=checkpoint_paths, model_proc=ModelProcessing.DEFAULT) with pytest.raises(Exception): ml_runner = MLRunner(config, get_default_azure_config()) ml_runner.register_segmentation_model( best_epoch=0, best_epoch_dice=0, checkpoint_paths=checkpoint_paths, model_proc=ModelProcessing.DEFAULT)
def test_image_encoder_with_segmentation( test_output_dirs: OutputFolderForTests, encode_channels_jointly: bool, aggregation_type: AggregationType, imaging_feature_type: ImagingFeatureType) -> None: """ Test if the image encoder networks can be trained on segmentations from HDF5. """ logging_to_stdout() set_random_seed(0) scan_size = (6, 64, 60) dataset_contents = """subject,channel,path,label S1,week0,scan1.h5, S1,week1,scan2.h5,True S2,week0,scan3.h5, S2,week1,scan4.h5,False S3,week0,scan5.h5, S3,week1,scan6.h5,True S4,week0,scan7.h5, S4,week1,scan8.h5,True """ config = ImageEncoder(encode_channels_jointly=encode_channels_jointly, imaging_feature_type=imaging_feature_type, should_validate=False, aggregation_type=aggregation_type, scan_size=scan_size) # This fails with 16bit precision, saying "torch.nn.functional.binary_cross_entropy and torch.nn.BCELoss are # unsafe to autocast. Many models use a sigmoid layer right before the binary cross entropy layer. In this case, # combine the two layers using torch.nn.functional.binary_cross_entropy_with_logits or # torch.nn.BCEWithLogitsLoss. binary_cross_entropy_with_logits and BCEWithLogits are safe to autocast." config.use_mixed_precision = False config.set_output_to(test_output_dirs.root_dir) config.num_epochs = 1 config.local_dataset = Path() config.dataset_data_frame = pd.read_csv(StringIO(dataset_contents), sep=",", dtype=str) # Patch the load_images function that will be called once we access a dataset item image_and_seg = ImageAndSegmentations[np.ndarray]( images=np.zeros(scan_size, dtype=np.float32), segmentations=np.ones(scan_size, dtype=np.uint8)) with mock.patch('InnerEye.ML.utils.io_util.load_image_in_known_formats', return_value=image_and_seg): azure_config = get_default_azure_config() azure_config.train = True MLRunner(config, azure_config).run()
def test_run_ml_with_segmentation_model( test_output_dirs: TestOutputDirectories) -> None: """ Test training and testing of segmentation models, when it is started together via run_ml. """ train_config = DummyModel() train_config.num_dataload_workers = 0 train_config.restrict_subjects = "1" # Increasing the test crop size should not have any effect on the results. # This is for a bug in an earlier version of the code where the wrong execution mode was used to # compute the expected mask size at training time. train_config.test_crop_size = (75, 75, 75) train_config.perform_training_set_inference = False train_config.perform_validation_and_test_set_inference = True train_config.set_output_to(test_output_dirs.root_dir) azure_config = get_default_azure_config() azure_config.train = True MLRunner(train_config, azure_config).run()
def test_image_encoder_with_segmentation( test_output_dirs: OutputFolderForTests, encode_channels_jointly: bool, aggregation_type: AggregationType, imaging_feature_type: ImagingFeatureType) -> None: """ Test if the image encoder networks can be trained on segmentations from HDF5. """ logging_to_stdout() set_random_seed(0) scan_size = (6, 64, 60) dataset_contents = """subject,channel,path,label S1,week0,scan1.h5, S1,week1,scan2.h5,True S2,week0,scan3.h5, S2,week1,scan4.h5,False S3,week0,scan5.h5, S3,week1,scan6.h5,True S4,week0,scan7.h5, S4,week1,scan8.h5,True """ config = ImageEncoder(encode_channels_jointly=encode_channels_jointly, imaging_feature_type=imaging_feature_type, should_validate=False, aggregation_type=aggregation_type, scan_size=scan_size) config.use_mixed_precision = True config.set_output_to(test_output_dirs.root_dir) config.num_epochs = 1 config.local_dataset = Path() config.dataset_data_frame = pd.read_csv(StringIO(dataset_contents), sep=",", dtype=str) # Patch the load_images function that will be called once we access a dataset item image_and_seg = ImageAndSegmentations[np.ndarray]( images=np.zeros(scan_size, dtype=np.float32), segmentations=np.ones(scan_size, dtype=np.uint8)) with mock.patch("InnerEye.ML.run_ml.is_offline_run_context", return_value=True): with mock.patch( 'InnerEye.ML.utils.io_util.load_image_in_known_formats', return_value=image_and_seg): azure_config = get_default_azure_config() azure_config.train = True MLRunner(config, azure_config=azure_config).run()
def test_run_ml_with_classification_model( test_output_dirs: TestOutputDirectories, number_of_offline_cross_validation_splits: int, number_of_cross_validation_splits_per_fold: int, model_name: str) -> None: """ Test training and testing of classification models, when it is started together via run_ml. """ logging_to_stdout() azure_config = get_default_azure_config() azure_config.train = True train_config: ScalarModelBase = ModelConfigLoader[ScalarModelBase]() \ .create_model_config_from_name(model_name) train_config.number_of_cross_validation_splits = number_of_offline_cross_validation_splits train_config.number_of_cross_validation_splits_per_fold = number_of_cross_validation_splits_per_fold train_config.set_output_to(test_output_dirs.root_dir) if train_config.perform_sub_fold_cross_validation: train_config.local_dataset = full_ml_test_data_path( "classification_data_sub_fold_cv") MLRunner(train_config, azure_config).run() _check_offline_cross_validation_output_files(train_config) if train_config.is_regression_model: assert (train_config.outputs_folder / "0" / "error_plot_4.png").is_file() if train_config.perform_cross_validation: # Test that the result files can be correctly picked up by the cross validation routine. # For that, we point the downloader to the local results folder. The core download method # recognizes run_recovery_id == None as the signal to read from the local_run_results folder. config_and_files = get_config_and_results_for_offline_runs( train_config) result_files = config_and_files.files # One file for VAL and one for TRAIN for each child run assert len( result_files ) == train_config.get_total_number_of_cross_validation_runs() * 2 for file in result_files: assert file.execution_mode != ModelExecutionMode.TEST assert file.dataset_csv_file is not None assert file.dataset_csv_file.exists() assert file.metrics_file is not None assert file.metrics_file.exists()
def test_runner_restart(test_output_dirs: OutputFolderForTests) -> None: """ Test if starting training from a folder where the checkpoints folder already has recovery checkpoints picks up that it is a recovery run. Also checks that we update the start epoch in the config at loading time. """ model_config = DummyClassification() model_config.set_output_to(test_output_dirs.root_dir) model_config.num_epochs = FIXED_EPOCH + 2 # We save all checkpoints - if recovery works as expected we should have a new checkpoint for epoch 4, 5. model_config.recovery_checkpoint_save_interval = 1 model_config.recovery_checkpoints_save_last_k = -1 runner = MLRunner(model_config=model_config) runner.setup(use_mount_or_download_dataset=False) # Epochs are 0 based for saving create_model_and_store_checkpoint(model_config, runner.container.checkpoint_folder / f"{RECOVERY_CHECKPOINT_FILE_NAME}_epoch=" f"{FIXED_EPOCH - 1}{CHECKPOINT_SUFFIX}", weights_only=False) azure_config = get_default_azure_config() checkpoint_handler = CheckpointHandler( azure_config=azure_config, container=runner.container, project_root=test_output_dirs.root_dir) _, storing_logger = model_train(checkpoint_handler=checkpoint_handler, container=runner.container) # We expect to have 4 checkpoints, FIXED_EPOCH (recovery), FIXED_EPOCH+1, FIXED_EPOCH and best. assert len(os.listdir(runner.container.checkpoint_folder)) == 4 assert (runner.container.checkpoint_folder / f"{RECOVERY_CHECKPOINT_FILE_NAME}_epoch=" f"{FIXED_EPOCH - 1}{CHECKPOINT_SUFFIX}").exists() assert (runner.container.checkpoint_folder / f"{RECOVERY_CHECKPOINT_FILE_NAME}_epoch=" f"{FIXED_EPOCH}{CHECKPOINT_SUFFIX}").exists() assert (runner.container.checkpoint_folder / f"{RECOVERY_CHECKPOINT_FILE_NAME}_epoch=" f"{FIXED_EPOCH + 1}{CHECKPOINT_SUFFIX}").exists() assert (runner.container.checkpoint_folder / BEST_CHECKPOINT_FILE_NAME_WITH_SUFFIX).exists() # Check that we really restarted epoch from epoch FIXED_EPOCH. assert list(storing_logger.epochs) == [FIXED_EPOCH, FIXED_EPOCH + 1] # type: ignore
def test_run_ml_with_multi_label_sequence_in_crossval(test_output_dirs: TestOutputDirectories) -> None: """ Test training and testing of sequence models that predicts at multiple time points, including aggregation of cross validation results. """ logging_to_stdout() config = ToyMultiLabelSequenceModel(should_validate=False) assert config.get_target_indices() == [1, 2, 3] expected_prediction_targets = ["Seq_pos 01", "Seq_pos 02", "Seq_pos 03"] target_indices = config.get_target_indices() assert target_indices assert len(target_indices) == len(expected_prediction_targets) config.set_output_to(test_output_dirs.root_dir) config.dataset_data_frame = _get_multi_label_sequence_dataframe() config.pre_process_dataset_dataframe() config.num_epochs = 1 config.number_of_cross_validation_splits = 2 azure_config = get_default_azure_config() azure_config.train = True MLRunner(config, azure_config).run()