def test_innereyecontainer_setup_passes_on_allow_incomplete_labels( test_output_dirs: OutputFolderForTests, allow_partial_ground_truth: bool) -> None: """ Test that InnerEyeContainer.setup passes on the correct value of allow_incomplete_labels to full_image_dataset.convert_channels_to_file_paths :param test_output_dirs: Test fixture. :param allow_partial_ground_truth: The value to set allow_incomplete_labels to and check it is passed through. """ config = DummyModel() config.set_output_to(test_output_dirs.root_dir) config.allow_incomplete_labels = allow_partial_ground_truth container = InnerEyeContainer(config) def mocked_convert_channels_to_file_paths( _: List[str], __: pd.DataFrame, ___: Path, ____: str, allow_incomplete_labels: bool) -> Tuple[List[Optional[Path]], str]: paths: List[Optional[Path]] = [] failed_channel_info = '' assert allow_incomplete_labels == allow_partial_ground_truth return paths, failed_channel_info with mock.patch("InnerEye.ML.lightning_base.convert_channels_to_file_paths" ) as convert_channels_to_file_paths_mock: convert_channels_to_file_paths_mock.side_effect = mocked_convert_channels_to_file_paths container.setup() convert_channels_to_file_paths_mock.assert_called()
def test_run_scoring(test_output_dirs: OutputFolderForTests, is_ensemble: bool) -> None: """ Run the scoring script on an image file. This test lives outside the normal Tests folder because it imports "score.py" from the repository root folder. If we switched to InnerEye as a package, we would have to treat this import special. The inference run here is on a 1-channel model, whereas test_register_and_score_model works with a 2-channel model. """ seed_everything(42) checkpoint = test_output_dirs.root_dir / "checkpoint.ckpt" image_size = (40, 40, 40) test_crop_size = image_size dummy_config = DummyModel() dummy_config.test_crop_size = test_crop_size dummy_config.inference_stride_size = (10, 10, 10) dummy_config.inference_batch_size = 10 create_model_and_store_checkpoint(dummy_config, checkpoint) all_paths = [checkpoint] * 2 if is_ensemble else [checkpoint] inference_pipeline, dummy_config = create_inference_pipeline(dummy_config, all_paths, use_gpu=False) image_with_header = io_util.load_nifti_image(test_image) image_with_header.image = image_with_header.image[:image_size[ 0], :image_size[1], :image_size[2]] result = run_inference([image_with_header, image_with_header], inference_pipeline, dummy_config) assert image_with_header.image.shape == result.shape # type: ignore print(f"Unique result values: {np.unique(result)}") assert np.all(result == 1)
def test_score_check_spacing() -> None: config = DummyModel() config.dataset_expected_spacing_xyz = (1.0, 1.0, 3.0) image_with_header = io_util.load_nifti_image(img_nii_path) spacing_xyz = reverse_tuple_float3(image_with_header.header.spacing) assert is_spacing_valid(spacing_xyz, config.dataset_expected_spacing_xyz) assert is_spacing_valid(spacing_xyz, (1, 1, 3.01)) assert not is_spacing_valid(spacing_xyz, (1, 1, 3.2))
def model_config( slice_exclusion_rules: List[SliceExclusionRule], summed_probability_rules: List[SummedProbabilityRule] ) -> SegmentationModelBase: test_config = DummyModel() test_config.slice_exclusion_rules = slice_exclusion_rules test_config.summed_probability_rules = summed_probability_rules test_config.ground_truth_ids = ground_truth_ids return test_config
def test_optim_params1(test_output_dirs: OutputFolderForTests) -> None: """ Test if the optimizer parameters are read correctly for InnerEye configs. """ model = DummyModel() model.set_output_to(test_output_dirs.root_dir) runner = MLRunner(model_config=model) runner.setup() lightning_model = runner.container.model optim, _ = lightning_model.configure_optimizers() assert optim[0].param_groups[0]["lr"] == 1e-3
def test_construct_loss_function() -> None: model_config = DummyModel() model_config.loss_type = SegmentationLoss.Mixture # Weights deliberately do not sum to 1.0. weights = [1.5, 0.5] model_config.mixture_loss_components = [ MixtureLossComponent(weights[0], SegmentationLoss.CrossEntropy, 0.2), MixtureLossComponent(weights[1], SegmentationLoss.SoftDice, 0.1)] loss_fn = ModelTrainingStepsForSegmentation.construct_loss_function(model_config) assert isinstance(loss_fn, MixtureLoss) assert len(loss_fn.components) == len(weights) assert loss_fn.components[0][0] == weights[0] / sum(weights) assert loss_fn.components[1][0] == weights[1] / sum(weights)
def test_score_image_dicom_mock_run_store( test_output_dirs: OutputFolderForTests) -> None: """ Test that dicom in and dicom-rt out works, by mocking out run and store functions. This mocks out run_inference and store_as_ubyte_nifti so that init_from_model_inference_json is tested in addition to the tests in test_score_image_dicom_mock_all. :param test_output_dirs: Test output directories. """ mock_segmentation = {'mock_segmentation': True} model_config = DummyModel() model_config.set_output_to(test_output_dirs.root_dir) checkpoint_path = model_config.checkpoint_folder / "checkpoint.ckpt" create_model_and_store_checkpoint(model_config, checkpoint_path) azure_config = AzureConfig() project_root = Path(__file__).parent.parent ml_runner = MLRunner(model_config=model_config, azure_config=azure_config, project_root=project_root) model_folder = test_output_dirs.root_dir / "final" ml_runner.copy_child_paths_to_folder(model_folder=model_folder, checkpoint_paths=[checkpoint_path]) zipped_dicom_series_path = test_output_dirs.root_dir / "temp_pack_dicom_series" / "dicom_series.zip" zip_known_dicom_series(zipped_dicom_series_path) score_pipeline_config = ScorePipelineConfig( data_folder=zipped_dicom_series_path.parent, model_folder=str(model_folder), image_files=[str(zipped_dicom_series_path)], result_image_name=HNSEGMENTATION_FILE.name, use_gpu=False, use_dicom=True, model_id="Dummy:1") with mock.patch('score.run_inference', return_value=mock_segmentation) as mock_run_inference: with mock.patch( 'score.store_as_ubyte_nifti', return_value=HNSEGMENTATION_FILE) as mock_store_as_ubyte_nifti: segmentation = score_image(score_pipeline_config) assert_zip_file_contents(segmentation, HN_DICOM_RT_ZIPPED, model_folder) mock_run_inference.assert_called() mock_store_as_ubyte_nifti.assert_called()
def test_save_and_load_state_dict(lr_scheduler_type: LRSchedulerType) -> None: def object_dict_same(lr1: SchedulerWithWarmUp, lr2: SchedulerWithWarmUp) -> bool: """ Tests to see if two LRScheduler objects are the same. This ignores lambdas if one of the schedulers is LambdaLR, since lambdas are not stored to the state dict. """ # ignore the _scheduler and _warmup objects, compare those separately dict1 = { key: val for key, val in lr1.__dict__.items() if key != "_scheduler" and key != "_warmup" } dict2 = { key: val for key, val in lr2.__dict__.items() if key != "_scheduler" and key != "_warmup" } # see if the underlying scheduler object is the same scheduler1_dict = { key: val for key, val in lr1._scheduler.__dict__.items() if key != "lr_lambdas" } scheduler2_dict = { key: val for key, val in lr2._scheduler.__dict__.items() if key != "lr_lambdas" } warmup1_dict = lr1._warmup.__dict__ warmup2_dict = lr2._warmup.__dict__ return dict1 == dict2 and scheduler1_dict == scheduler2_dict and warmup1_dict == warmup2_dict config = DummyModel(num_epochs=10, l_rate_scheduler=lr_scheduler_type, l_rate_exponential_gamma=0.9, l_rate_step_gamma=0.9, l_rate_step_step_size=2, l_rate_multi_step_gamma=0.9, l_rate_multi_step_milestones=[3, 5, 7], l_rate_polynomial_gamma=0.9, l_rate_warmup=LRWarmUpType.Linear, l_rate_warmup_epochs=4) lr_scheduler_1, optimizer = _create_lr_scheduler_and_optimizer(config) lr_scheduler_1.step() # This is not supported functionality - we are doing this just to change _scheduler from its default state lr_scheduler_1._scheduler.step() lr_scheduler_1._scheduler.step() state_dict = lr_scheduler_1.state_dict() lr_scheduler_2, _ = _create_lr_scheduler_and_optimizer(config, optimizer) assert not object_dict_same(lr_scheduler_1, lr_scheduler_2) lr_scheduler_2.load_state_dict(state_dict) assert object_dict_same(lr_scheduler_1, lr_scheduler_2)
def test_run_scoring(is_ensemble: bool) -> None: checkpoints_paths = checkpoint_full_paths * 2 if is_ensemble else checkpoint_full_paths dummy_config = DummyModel() inference_pipeline, dummy_config = create_inference_pipeline(dummy_config, checkpoints_paths, use_gpu=False) image_with_header = io_util.load_nifti_image(test_image) result = run_inference([image_with_header, image_with_header], inference_pipeline, dummy_config) assert np.all(result == 1) assert image_with_header.image.shape == result.shape # type: ignore
def test_create_lr_scheduler_last_epoch() -> None: """ Test to check if the lr scheduler is initialized to the correct epoch """ l_rate = 1e-3 gamma = 0.5 total_epochs = 5 expected_lrs_per_epoch = [l_rate * (gamma**i) for i in range(total_epochs)] config = DummyModel() config.l_rate = l_rate config.l_rate_scheduler = LRSchedulerType.Step config.l_rate_step_step_size = 1 config.l_rate_step_gamma = gamma # create lr scheduler initial_scheduler, initial_optimizer = _create_lr_scheduler_and_optimizer( config) # check lr scheduler initialization step initial_epochs = 3 assert np.allclose(enumerate_scheduler(initial_scheduler, initial_epochs), expected_lrs_per_epoch[:initial_epochs]) # create lr scheduler for recovery checkpoint config.start_epoch = initial_epochs recovery_scheduler, recovery_optimizer = _create_lr_scheduler_and_optimizer( config) # Both the scheduler and the optimizer need to be loaded from the checkpoint. recovery_scheduler.load_state_dict(initial_scheduler.state_dict()) recovery_optimizer.load_state_dict(initial_optimizer.state_dict()) assert recovery_scheduler.last_epoch == config.start_epoch # check lr scheduler initialization matches the checkpoint epoch # as training will start for start_epoch + 1 in this case assert np.allclose(enumerate_scheduler(recovery_scheduler, 2), expected_lrs_per_epoch[initial_epochs:])
def test_download_azureml_dataset(test_output_dirs: OutputFolderForTests) -> None: dataset_name = "test-dataset" config = DummyModel() config.local_dataset = None config.azure_dataset_id = "" azure_config = get_default_azure_config() runner = MLRunner(config, azure_config=azure_config) # If the model has neither local_dataset or azure_dataset_id, mount_or_download_dataset should fail. # This mounting call must happen before any other operations on the container, because already the model # creation may need access to the dataset. with pytest.raises(ValueError) as ex: runner.setup() assert ex.value.args[0] == "The model must contain either local_dataset or azure_dataset_id." runner.project_root = test_output_dirs.root_dir # Pointing the model to a dataset folder that does not exist should raise an Exception fake_folder = runner.project_root / "foo" runner.container.local_dataset = fake_folder with pytest.raises(FileNotFoundError): runner.mount_or_download_dataset(runner.container.azure_dataset_id, runner.container.local_dataset) # If the local dataset folder exists, mount_or_download_dataset should not do anything. fake_folder.mkdir() local_dataset = runner.mount_or_download_dataset(runner.container.azure_dataset_id, runner.container.local_dataset) assert local_dataset == fake_folder # Pointing the model to a dataset in Azure should trigger a download runner.container.local_dataset = None runner.container.azure_dataset_id = dataset_name with logging_section("Starting download"): result_path = runner.mount_or_download_dataset(runner.container.azure_dataset_id, runner.container.local_dataset) # Download goes into <project_root> / "datasets" / "test_dataset" expected_path = runner.project_root / fixed_paths.DATASETS_DIR_NAME / dataset_name assert result_path == expected_path assert result_path.is_dir() dataset_csv = Path(result_path) / DATASET_CSV_FILE_NAME assert dataset_csv.is_file() # Check that each individual file in the dataset is present for folder in [1, *range(10, 20)]: sub_folder = result_path / str(folder) sub_folder.is_dir() for file in ["ct", "esophagus", "heart", "lung_l", "lung_r", "spinalcord"]: f = (sub_folder / file).with_suffix(".nii.gz") assert f.is_file()
def test_file_system_with_subfolders( test_output_dirs: OutputFolderForTests) -> None: """ Test if a subfolder can be created within the output folder structure, for use with cross validation. """ model = DummyModel() model.set_output_to(test_output_dirs.root_dir) container = InnerEyeContainer(model) # File system should be copied from model config to container assert container.file_system_config == model.file_system_config runner = MLRunner(model_config=model) runner.setup() assert str(runner.container.outputs_folder).endswith(model.model_name) output_subfolder = "foo" expected_folder = runner.container.outputs_folder / output_subfolder runner = MLRunner(model_config=model, output_subfolder=output_subfolder) runner.setup() assert runner.container.outputs_folder == expected_folder
def test_score_image_dicom_mock_run( test_output_dirs: OutputFolderForTests) -> None: """ Test that dicom in and dicom-rt out works, by mocking out only the run scoring function. This mocks out run_inference so that store_as_ubyte_nifti is tested in addition to the tests in test_score_image_dicom_mock_run_store. :param test_output_dirs: Test output directories. """ model_config = DummyModel() model_config.set_output_to(test_output_dirs.root_dir) checkpoint_path = model_config.checkpoint_folder / "checkpoint.ckpt" create_model_and_store_checkpoint(model_config, checkpoint_path) azure_config = AzureConfig() project_root = Path(__file__).parent.parent ml_runner = MLRunner(model_config=model_config, azure_config=azure_config, project_root=project_root) model_folder = test_output_dirs.root_dir / "final" ml_runner.copy_child_paths_to_folder(model_folder=model_folder, checkpoint_paths=[checkpoint_path]) zipped_dicom_series_path = zip_dicom_series(model_folder) score_pipeline_config = ScorePipelineConfig( data_folder=zipped_dicom_series_path.parent, model_folder=str(model_folder), image_files=[str(zipped_dicom_series_path)], result_image_name=HNSEGMENTATION_FILE.name, use_gpu=False, use_dicom=True) image_with_header = io_util.load_nifti_image(HNSEGMENTATION_FILE) with mock.patch( 'score.run_inference', return_value=image_with_header.image) as mock_run_inference: segmentation = score_image(score_pipeline_config) assert_zip_file_contents(segmentation, HN_DICOM_RT_ZIPPED, model_folder) mock_run_inference.assert_called()
def test_model_name_for_innereye_container() -> None: """ Test if the InnerEye container picks up the name of the model correctly. The name will impact the output folder structure that is created. """ expected_name = "DummyModel" model = DummyModel() assert model.model_name == expected_name container = InnerEyeContainer(model) assert container.model_name == expected_name
def test_run_ml_with_segmentation_model( test_output_dirs: OutputFolderForTests) -> None: """ Test training and testing of segmentation models, when it is started together via run_ml. """ config = DummyModel() config.num_dataload_workers = 0 config.restrict_subjects = "1" # Increasing the test crop size should not have any effect on the results. # This is for a bug in an earlier version of the code where the wrong execution mode was used to # compute the expected mask size at training time. config.test_crop_size = (75, 75, 75) config.inference_on_train_set = False config.inference_on_val_set = True config.inference_on_test_set = True config.set_output_to(test_output_dirs.root_dir) azure_config = get_default_azure_config() azure_config.train = True MLRunner(config, azure_config=azure_config).run()
def test_download_model_weights( test_output_dirs: OutputFolderForTests) -> None: # Download a sample ResNet model from a URL given in the Pytorch docs # The downloaded model does not match the architecture, which is okay since we are only testing the download here. model_config = DummyModel(weights_url=EXTERNAL_WEIGHTS_URL_EXAMPLE) manage_recovery = get_default_checkpoint_handler( model_config=model_config, project_root=test_output_dirs.root_dir) result_path = manage_recovery.download_weights() assert result_path.is_file()
def test_visualize_commandline1() -> None: """ Testing for a bug in commandline processing: The model configuration was always overwritten with all the default values of each field in Config, rather than only the overrides specified on the commandline. :return: """ default_config = DummyModel() old_photonorm = default_config.norm_method old_random_seed = default_config.get_effective_random_seed() new_dataset = "new_dataset" assert default_config.azure_dataset_id != new_dataset with mock.patch("sys.argv", ["", f"--azure_dataset_id={new_dataset}"]): updated_config, runner_config, _ = get_configs(default_config, yaml_file_path=fixed_paths.SETTINGS_YAML_FILE) assert updated_config.azure_dataset_id == new_dataset # These two values were not specified on the commandline, and should be at their original values. assert updated_config.norm_method == old_photonorm assert updated_config.get_effective_random_seed() == old_random_seed # Credentials and variables should have been picked up from yaml files assert len(runner_config.datasets_container) > 0
def test_single_element() -> None: config = DummyModel() element = ModelTrainingStepsForSegmentation.construct_non_mixture_loss_function( config, SegmentationLoss.CrossEntropy, power=None) mixture = MixtureLoss([(1.0, element)]) target = torch.tensor([[[0, 0, 1], [1, 1, 0]]], dtype=torch.float32) logits = torch.tensor([[[-1e9, -1e9, 0], [0, 0, 0]]], dtype=torch.float32) # Extract class indices element_loss = element(logits, target) mixture_loss = mixture(logits, target) assert torch.isclose(element_loss, mixture_loss)
def create_data_loaders(train_config: DummyModel) -> None: train_config.train_batch_size = 1 train_config.local_dataset = base_path # create the dataset splits dataset_splits = train_config.get_dataset_splits() # create the data loaders data_loaders = train_config.create_data_loaders() train_loader = data_loaders[ModelExecutionMode.TRAIN] val_loader = data_loaders[ModelExecutionMode.VAL] def check_patient_id_in_dataset(loader: DataLoader, split: pd.DataFrame) -> None: subjects = list(split.subject.unique()) for i, x in enumerate(loader): sample_from_loader = CroppedSample.from_dict(x) assert isinstance(sample_from_loader.metadata, list) assert len(sample_from_loader.metadata) == 1 assert sample_from_loader.metadata[0].patient_id in subjects # check if the subjects in the dataloaders are the same in the corresponding dataset splits for loader, split in [(train_loader, dataset_splits.train), (val_loader, dataset_splits.val)]: check_patient_id_in_dataset(loader, split)
def test_multistep_lr() -> None: l_rate = 0.3 config = DummyModel(l_rate_scheduler=LRSchedulerType.MultiStep, l_rate=l_rate, l_rate_multi_step_gamma=0.1, num_epochs=10, l_rate_multi_step_milestones=[2], l_rate_warmup=LRWarmUpType.Linear, l_rate_warmup_epochs=5) def check_warmup(expected: List[float]) -> None: scheduler, _ = _create_lr_scheduler_and_optimizer(config) actual = enumerate_scheduler(scheduler, 4) assert actual == expected # No warmup: multi-step LR with milestone after 2 epochs original_schedule = [l_rate, l_rate, l_rate * 0.1, l_rate * 0.1] config.l_rate_warmup = LRWarmUpType.Linear config.l_rate_warmup_epochs = 0 check_warmup(original_schedule) # 1 epoch warmup: linear function up to the initial learning rate gives a warmup value of half the initial LR config.l_rate_warmup_epochs = 1 check_warmup([l_rate * 0.5] + original_schedule[:3]) # 2 epochs warmup config.l_rate_warmup_epochs = 2 check_warmup([l_rate / 3, l_rate * 2 / 3] + original_schedule[:2])
def test_resume_from_saved_state(lr_scheduler_type: LRSchedulerType, warmup_epochs: int) -> None: """ Tests if LR scheduler when restarted from an epoch continues as expected. """ restart_from_epoch = 4 config = DummyModel(num_epochs=7, l_rate_scheduler=lr_scheduler_type, l_rate_exponential_gamma=0.9, l_rate_step_gamma=0.9, l_rate_step_step_size=2, l_rate_multi_step_gamma=0.9, l_rate_multi_step_milestones=[3, 5, 7], l_rate_polynomial_gamma=0.9, l_rate_warmup=LRWarmUpType.Linear if warmup_epochs > 0 else LRWarmUpType.NoWarmUp, l_rate_warmup_epochs=warmup_epochs) # This scheduler mimics what happens if we train for the full set of epochs scheduler_all_epochs, _ = _create_lr_scheduler_and_optimizer(config) expected_lr_list = enumerate_scheduler(scheduler_all_epochs, config.num_epochs) # Create a scheduler where training will be recovered scheduler1, optimizer1 = _create_lr_scheduler_and_optimizer(config) # Scheduler 1 is only run for 4 epochs, and then "restarted" to train the rest of the epochs. result_lr_list = enumerate_scheduler(scheduler1, restart_from_epoch) # resume state: This just means setting start_epoch in the config config.start_epoch = restart_from_epoch scheduler_resume, optimizer_resume = _create_lr_scheduler_and_optimizer( config) # Load a "checkpoint" for both scheduler and optimizer scheduler_resume.load_state_dict(scheduler1.state_dict()) optimizer_resume.load_state_dict(optimizer1.state_dict()) result_lr_list.extend( enumerate_scheduler(scheduler_resume, config.num_epochs - restart_from_epoch)) print(f"Actual schedule: {result_lr_list}") print(f"Expected schedule: {expected_lr_list}") assert len(result_lr_list) == len(expected_lr_list) assert np.allclose(result_lr_list, expected_lr_list)
def test_cosine_decay_function() -> None: """ Tests Cosine lr decay function at (pi/2) and verifies if the value is correct. """ config = DummyModel(l_rate_scheduler=LRSchedulerType.Cosine, num_epochs=10, min_l_rate=0.0) # create lr scheduler test_epoch = 5 lr_scheduler, _ = _create_lr_scheduler_and_optimizer(config) for _ in range(test_epoch): lr_scheduler.step() assert lr_scheduler.get_last_lr()[0] == 0.5 * config.l_rate
def _test_mount_for_lightning_container(test_output_dirs: OutputFolderForTests, is_offline_run: bool, local_dataset: Optional[Path], azure_dataset: str, is_lightning_model: bool) -> LightningContainer: config: Optional[DeepLearningConfig] = None container: Optional[LightningContainer] = None if is_lightning_model: container = DummyContainerWithDatasets() container.azure_dataset_id = azure_dataset container.local_dataset = local_dataset else: config = DummyModel() config.azure_dataset_id = azure_dataset config.local_dataset = local_dataset # The legacy InnerEye models require an existing dataset_csv file present in the dataset folder. Create that. download_path = test_output_dirs.root_dir / "downloaded" mount_path = test_output_dirs.root_dir / "mounted" if not is_lightning_model: train_and_test_data = "train_and_test_data" for path in [download_path, mount_path, test_output_dirs.root_dir]: # If destination folder exists, delete content to ensure consistency and avoid 'FileExistsError' if (path / train_and_test_data).is_dir(): shutil.rmtree(path / train_and_test_data) # Creates directory structure and copy data shutil.copytree(full_ml_test_data_path(train_and_test_data), path / train_and_test_data) # Copy 'data.csv' file shutil.copy(full_ml_test_data_path(DATASET_CSV_FILE_NAME), path / DATASET_CSV_FILE_NAME) with mock.patch("InnerEye.ML.run_ml.MLRunner.is_offline_run", is_offline_run): with mock.patch("InnerEye.ML.run_ml.download_dataset", return_value=download_path): with mock.patch("InnerEye.ML.run_ml.try_to_mount_input_dataset", return_value=mount_path): runner = MLRunner(config, container=container, azure_config=None, project_root=test_output_dirs.root_dir) runner.setup() return runner.container
def test_two_elements() -> None: config = DummyModel() element1 = ModelTrainingStepsForSegmentation.construct_non_mixture_loss_function( config, SegmentationLoss.CrossEntropy, power=None) element2 = ModelTrainingStepsForSegmentation.construct_non_mixture_loss_function( config, SegmentationLoss.SoftDice, power=None) weight1, weight2 = 0.3, 0.7 mixture = MixtureLoss([(weight1, element1), (weight2, element2)]) target = torch.tensor([[[0, 0, 1], [1, 1, 0]]], dtype=torch.float32) logits = torch.tensor([[[-1e9, -1e9, 0], [0, 0, 0]]], dtype=torch.float32) # Extract class indices element1_loss = element1(logits, target) element2_loss = element2(logits, target) mixture_loss = mixture(logits, target) assert torch.isclose(weight1 * element1_loss + weight2 * element2_loss, mixture_loss)
def test_model_inference_train_and_test( test_output_dirs: OutputFolderForTests, perform_cross_validation: bool, perform_training_set_inference: bool) -> None: config = DummyModel() config.number_of_cross_validation_splits = 2 if perform_cross_validation else 0 config.perform_training_set_inference = perform_training_set_inference # Plotting crashes with random TCL errors on Windows, disable that for Windows PR builds. config.is_plotting_enabled = common_util.is_linux() config.set_output_to(test_output_dirs.root_dir) config.local_dataset = full_ml_test_data_path() # To make it seem like there was a training run before this, copy checkpoints into the checkpoints folder. stored_checkpoints = full_ml_test_data_path("checkpoints") shutil.copytree(str(stored_checkpoints), str(config.checkpoint_folder)) checkpoint_handler = get_default_checkpoint_handler( model_config=config, project_root=test_output_dirs.root_dir) checkpoint_handler.additional_training_done() result, _, _ = MLRunner(config).model_inference_train_and_test( checkpoint_handler=checkpoint_handler) if result is None: raise ValueError("Error result cannot be None") assert isinstance(result, InferenceMetricsForSegmentation) for key, _ in result.epochs.items(): epoch_folder_name = common_util.epoch_folder_name(key) for folder in [ ModelExecutionMode.TRAIN.value, ModelExecutionMode.VAL.value, ModelExecutionMode.TEST.value ]: results_folder = config.outputs_folder / epoch_folder_name / folder folder_exists = results_folder.is_dir() if folder in [ ModelExecutionMode.TRAIN.value, ModelExecutionMode.VAL.value ]: if perform_training_set_inference: assert folder_exists else: assert folder_exists
def test_lr_monotonically_decreasing_function( lr_scheduler_type: LRSchedulerType) -> None: """ Tests if LR scheduler is a monotonically decreasing function """ config = DummyModel(num_epochs=10, l_rate_scheduler=lr_scheduler_type, l_rate_exponential_gamma=0.9, l_rate_step_gamma=0.9, l_rate_step_step_size=1, l_rate_multi_step_gamma=0.9, l_rate_multi_step_milestones=[3, 5, 7], l_rate_polynomial_gamma=0.9) def non_increasing(L: List) -> bool: return all(x >= y for x, y in zip(L, L[1:])) # create lr scheduler lr_scheduler, _ = _create_lr_scheduler_and_optimizer(config) lr_list = enumerate_scheduler(lr_scheduler, config.num_epochs) assert non_increasing(lr_list)
def test_model_inference_train_and_test( test_output_dirs: TestOutputDirectories, perform_cross_validation: bool, perform_training_set_inference: bool) -> None: config = DummyModel() config.number_of_cross_validation_splits = 2 if perform_cross_validation else 0 config.perform_training_set_inference = perform_training_set_inference # Plotting crashes with random TCL errors on Windows, disable that for Windows PR builds. config.is_plotting_enabled = common_util.is_linux() config.set_output_to(test_output_dirs.root_dir) config.local_dataset = full_ml_test_data_path() # Mimic the behaviour that checkpoints are downloaded from blob storage into the checkpoints folder. stored_checkpoints = full_ml_test_data_path("checkpoints") shutil.copytree(str(stored_checkpoints), str(config.checkpoint_folder)) result, _, _ = MLRunner(config).model_inference_train_and_test() if result is None: raise ValueError("Error result cannot be None") assert isinstance(result, InferenceMetricsForSegmentation) for key, _ in result.epochs.items(): epoch_folder_name = common_util.epoch_folder_name(key) for folder in [ ModelExecutionMode.TRAIN.value, ModelExecutionMode.VAL.value, ModelExecutionMode.TEST.value ]: results_folder = config.outputs_folder / epoch_folder_name / folder folder_exists = results_folder.is_dir() if folder in [ ModelExecutionMode.TRAIN.value, ModelExecutionMode.VAL.value ]: if perform_training_set_inference: assert folder_exists else: assert folder_exists
def test_model_inference_train_and_test( test_output_dirs: OutputFolderForTests, perform_cross_validation: bool, perform_training_set_inference: bool) -> None: config = DummyModel() config.number_of_cross_validation_splits = 2 if perform_cross_validation else 0 config.perform_training_set_inference = perform_training_set_inference # Plotting crashes with random TCL errors on Windows, disable that for Windows PR builds. config.is_plotting_enabled = common_util.is_linux() config.set_output_to(test_output_dirs.root_dir) config.local_dataset = full_ml_test_data_path() checkpoint_path = config.checkpoint_folder / BEST_CHECKPOINT_FILE_NAME_WITH_SUFFIX create_model_and_store_checkpoint(config, checkpoint_path) checkpoint_handler = get_default_checkpoint_handler( model_config=config, project_root=test_output_dirs.root_dir) checkpoint_handler.additional_training_done() result, _, _ = MLRunner(config).model_inference_train_and_test( checkpoint_handler=checkpoint_handler) if result is None: raise ValueError("Error result cannot be None") assert isinstance(result, InferenceMetricsForSegmentation) epoch_folder_name = common_util.BEST_EPOCH_FOLDER_NAME for folder in [ ModelExecutionMode.TRAIN.value, ModelExecutionMode.VAL.value, ModelExecutionMode.TEST.value ]: results_folder = config.outputs_folder / epoch_folder_name / folder folder_exists = results_folder.is_dir() if folder in [ ModelExecutionMode.TRAIN.value, ModelExecutionMode.VAL.value ]: if perform_training_set_inference: assert folder_exists else: assert folder_exists
def test_is_offline_cross_val_parent_run(offline_parent_cv_run: bool) -> None: train_config = DummyModel() train_config.number_of_cross_validation_splits = 2 if offline_parent_cv_run else 0 assert MLRunner(train_config).is_offline_cross_val_parent_run( ) == offline_parent_cv_run
def default_config() -> ModelConfigBase: config = DummyModel() config.set_output_to(str(full_ml_test_data_path("outputs"))) return config