def test_run_ml_with_classification_model( test_output_dirs: OutputFolderForTests, number_of_offline_cross_validation_splits: int, model_name: str) -> None: """ Test training and testing of classification models, when it is started together via run_ml. """ logging_to_stdout() azure_config = get_default_azure_config() azure_config.train = True config: ScalarModelBase = ModelConfigLoader[ScalarModelBase]() \ .create_model_config_from_name(model_name) config.number_of_cross_validation_splits = number_of_offline_cross_validation_splits config.set_output_to(test_output_dirs.root_dir) # Trying to run DDP from the test suite hangs, hence restrict to single GPU. config.max_num_gpus = 1 MLRunner(config, azure_config).run() _check_offline_cross_validation_output_files(config) if config.perform_cross_validation: # Test that the result files can be correctly picked up by the cross validation routine. # For that, we point the downloader to the local results folder. The core download method # recognizes run_recovery_id == None as the signal to read from the local_run_results folder. config_and_files = get_config_and_results_for_offline_runs(config) result_files = config_and_files.files # One file for VAL and one for TRAIN for each child run assert len(result_files ) == config.get_total_number_of_cross_validation_runs() * 2 for file in result_files: assert file.execution_mode != ModelExecutionMode.TEST assert file.dataset_csv_file is not None assert file.dataset_csv_file.exists() assert file.metrics_file is not None assert file.metrics_file.exists()
def spawn_offline_cross_val_classification_child_runs(self) -> None: """ Trains and Tests k models based on their respective data splits sequentially. Stores the results on the Validation set to the outputs directory of the parent run. """ _config = self.model_config assert isinstance(_config, ScalarModelBase) parent_run_file_system = _config.file_system_config def _spawn_run(cross_val_split_index: int) -> None: split_model_config = copy.deepcopy(_config) assert isinstance(split_model_config, ScalarModelBase) split_model_config.cross_validation_split_index = cross_val_split_index _local_split_folder_name = str(cross_val_split_index) split_model_config.file_system_config = parent_run_file_system.add_subfolder(_local_split_folder_name) logging.info(f"Running model train and test on cross validation split: {cross_val_split_index}") split_ml_runner = MLRunner(model_config=split_model_config, azure_config=self.azure_config, project_root=self.project_root, post_cross_validation_hook=self.post_cross_validation_hook, model_deployment_hook=self.model_deployment_hook) split_ml_runner.run() for i in range(_config.number_of_cross_validation_splits): _spawn_run(i) config_and_files = get_config_and_results_for_offline_runs(self.model_config) plot_cross_validation_from_files(config_and_files, Path(config_and_files.config.outputs_directory))
def spawn_offline_cross_val_classification_child_runs(self) -> None: """ Trains and Tests k models based on their respective data splits sequentially. Stores the results on the Validation set to the outputs directory of the parent run. """ _config = self.model_config assert isinstance(_config, ScalarModelBase) parent_run_file_system = _config.file_system_config def _spawn_run(cross_val_split_index: int, cross_val_sub_fold_split_index: int) -> None: split_model_config = copy.deepcopy(_config) assert isinstance(split_model_config, ScalarModelBase) split_model_config.cross_validation_split_index = cross_val_split_index split_model_config.cross_validation_sub_fold_split_index = cross_val_sub_fold_split_index if cross_val_sub_fold_split_index == DEFAULT_CROSS_VALIDATION_SPLIT_INDEX: _local_split_folder_name = str(cross_val_split_index) else: _local_split_folder_name = \ str((cross_val_split_index * split_model_config.number_of_cross_validation_splits_per_fold) + cross_val_sub_fold_split_index) split_model_config.file_system_config = parent_run_file_system.add_subfolder( _local_split_folder_name) logging.info( f"Running model train and test on cross validation split: {x}") split_ml_runner = MLRunner(split_model_config, self.azure_config, self.project_root, self.model_deployment_hook, self.innereye_submodule_name) split_ml_runner.run() cv_fold_indices = [ list(range(_config.number_of_cross_validation_splits_per_fold)) if _config.perform_sub_fold_cross_validation else [DEFAULT_CROSS_VALIDATION_SPLIT_INDEX] ] cv_fold_indices *= _config.number_of_cross_validation_splits for i, x in enumerate(cv_fold_indices): for y in x: _spawn_run(i, int(y)) config_and_files = get_config_and_results_for_offline_runs( self.model_config) plot_cross_validation_from_files( config_and_files, Path(config_and_files.config.outputs_directory))
def test_run_ml_with_classification_model( test_output_dirs: TestOutputDirectories, number_of_offline_cross_validation_splits: int, number_of_cross_validation_splits_per_fold: int, model_name: str) -> None: """ Test training and testing of classification models, when it is started together via run_ml. """ logging_to_stdout() azure_config = get_default_azure_config() azure_config.train = True train_config: ScalarModelBase = ModelConfigLoader[ScalarModelBase]() \ .create_model_config_from_name(model_name) train_config.number_of_cross_validation_splits = number_of_offline_cross_validation_splits train_config.number_of_cross_validation_splits_per_fold = number_of_cross_validation_splits_per_fold train_config.set_output_to(test_output_dirs.root_dir) if train_config.perform_sub_fold_cross_validation: train_config.local_dataset = full_ml_test_data_path( "classification_data_sub_fold_cv") MLRunner(train_config, azure_config).run() _check_offline_cross_validation_output_files(train_config) if train_config.is_regression_model: assert (train_config.outputs_folder / "0" / "error_plot_4.png").is_file() if train_config.perform_cross_validation: # Test that the result files can be correctly picked up by the cross validation routine. # For that, we point the downloader to the local results folder. The core download method # recognizes run_recovery_id == None as the signal to read from the local_run_results folder. config_and_files = get_config_and_results_for_offline_runs( train_config) result_files = config_and_files.files # One file for VAL and one for TRAIN for each child run assert len( result_files ) == train_config.get_total_number_of_cross_validation_runs() * 2 for file in result_files: assert file.execution_mode != ModelExecutionMode.TEST assert file.dataset_csv_file is not None assert file.dataset_csv_file.exists() assert file.metrics_file is not None assert file.metrics_file.exists()
def test_model_test(test_output_dirs: OutputFolderForTests) -> None: train_and_test_data_dir = full_ml_test_data_path("train_and_test_data") config = DummyModel() config.set_output_to(test_output_dirs.root_dir) epoch = 1 config.num_epochs = epoch assert config.get_test_epochs() == [epoch] placeholder_dataset_id = "place_holder_dataset_id" config.azure_dataset_id = placeholder_dataset_id transform = config.get_full_image_sample_transforms().test df = pd.read_csv(full_ml_test_data_path(DATASET_CSV_FILE_NAME)) df = df[df.subject.isin([1, 2])] # noinspection PyTypeHints config._datasets_for_inference = \ {ModelExecutionMode.TEST: FullImageDataset(config, df, full_image_sample_transforms=transform)} # type: ignore execution_mode = ModelExecutionMode.TEST checkpoint_handler = get_default_checkpoint_handler(model_config=config, project_root=test_output_dirs.root_dir) # Mimic the behaviour that checkpoints are downloaded from blob storage into the checkpoints folder. stored_checkpoints = full_ml_test_data_path("checkpoints") shutil.copytree(str(stored_checkpoints), str(config.checkpoint_folder)) checkpoint_handler.additional_training_done() inference_results = model_testing.segmentation_model_test(config, data_split=execution_mode, checkpoint_handler=checkpoint_handler) epoch_dir = config.outputs_folder / get_epoch_results_path(epoch, execution_mode) assert inference_results.epochs[epoch] == pytest.approx(0.66606902, abs=1e-6) assert config.outputs_folder.is_dir() assert epoch_dir.is_dir() patient1 = io_util.load_nifti_image(train_and_test_data_dir / "id1_channel1.nii.gz") patient2 = io_util.load_nifti_image(train_and_test_data_dir / "id2_channel1.nii.gz") assert_file_contains_string(epoch_dir / DATASET_ID_FILE, placeholder_dataset_id) assert_file_contains_string(epoch_dir / GROUND_TRUTH_IDS_FILE, "region") assert_text_files_match(epoch_dir / model_testing.METRICS_FILE_NAME, train_and_test_data_dir / model_testing.METRICS_FILE_NAME) assert_text_files_match(epoch_dir / model_testing.METRICS_AGGREGATES_FILE, train_and_test_data_dir / model_testing.METRICS_AGGREGATES_FILE) # Plotting results vary between platforms. Can only check if the file is generated, but not its contents. assert (epoch_dir / model_testing.BOXPLOT_FILE).exists() assert_nifti_content(epoch_dir / "001" / "posterior_region.nii.gz", get_image_shape(patient1), patient1.header, [136], np.ubyte) assert_nifti_content(epoch_dir / "002" / "posterior_region.nii.gz", get_image_shape(patient2), patient2.header, [136], np.ubyte) assert_nifti_content(epoch_dir / "001" / DEFAULT_RESULT_IMAGE_NAME, get_image_shape(patient1), patient1.header, [1], np.ubyte) assert_nifti_content(epoch_dir / "002" / DEFAULT_RESULT_IMAGE_NAME, get_image_shape(patient2), patient2.header, [1], np.ubyte) assert_nifti_content(epoch_dir / "001" / "posterior_background.nii.gz", get_image_shape(patient1), patient1.header, [118], np.ubyte) assert_nifti_content(epoch_dir / "002" / "posterior_background.nii.gz", get_image_shape(patient2), patient2.header, [118], np.ubyte) thumbnails_folder = epoch_dir / model_testing.THUMBNAILS_FOLDER assert thumbnails_folder.is_dir() png_files = list(thumbnails_folder.glob("*.png")) overlays = [f for f in png_files if "_region_slice_" in str(f)] assert len(overlays) == len(df.subject.unique()), "There should be one overlay/contour file per subject" # Writing dataset.csv normally happens at the beginning of training, # but this test reads off a saved checkpoint file. # Dataset.csv must be present for plot_cross_validation. config.write_dataset_files() # Test if the metrics files can be picked up correctly by the cross validation code config_and_files = get_config_and_results_for_offline_runs(config) result_files = config_and_files.files assert len(result_files) == 1 for file in result_files: assert file.execution_mode == execution_mode assert file.dataset_csv_file is not None assert file.dataset_csv_file.exists() assert file.metrics_file is not None assert file.metrics_file.exists()
def test_model_test(test_output_dirs: OutputFolderForTests, use_partial_ground_truth: bool, allow_partial_ground_truth: bool) -> None: """ Check the CSVs (and image files) output by InnerEye.ML.model_testing.segmentation_model_test :param test_output_dirs: The fixture in conftest.py :param use_partial_ground_truth: Whether to remove some ground truth labels from some test users :param allow_partial_ground_truth: What to set the allow_incomplete_labels flag to """ train_and_test_data_dir = full_ml_test_data_path("train_and_test_data") seed_everything(42) config = DummyModel() config.allow_incomplete_labels = allow_partial_ground_truth config.set_output_to(test_output_dirs.root_dir) placeholder_dataset_id = "place_holder_dataset_id" config.azure_dataset_id = placeholder_dataset_id transform = config.get_full_image_sample_transforms().test df = pd.read_csv(full_ml_test_data_path(DATASET_CSV_FILE_NAME)) if use_partial_ground_truth: config.check_exclusive = False config.ground_truth_ids = ["region", "region_1"] # As in Tests.ML.pipelines.test.inference.test_evaluate_model_predictions patients 3, 4, # and 5 are in the test dataset with: # Patient 3 has one missing ground truth channel: "region" df = df[df["subject"].ne(3) | df["channel"].ne("region")] # Patient 4 has all missing ground truth channels: "region", "region_1" df = df[df["subject"].ne(4) | df["channel"].ne("region")] df = df[df["subject"].ne(4) | df["channel"].ne("region_1")] # Patient 5 has no missing ground truth channels. config.dataset_data_frame = df df = df[df.subject.isin([3, 4, 5])] config.train_subject_ids = ['1', '2'] config.test_subject_ids = ['3', '4', '5'] config.val_subject_ids = ['6', '7'] else: df = df[df.subject.isin([1, 2])] if use_partial_ground_truth and not allow_partial_ground_truth: with pytest.raises(ValueError) as value_error: # noinspection PyTypeHints config._datasets_for_inference = { ModelExecutionMode.TEST: FullImageDataset(config, df, full_image_sample_transforms=transform) } # type: ignore assert "Patient 3 does not have channel 'region'" in str( value_error.value) return else: # noinspection PyTypeHints config._datasets_for_inference = { ModelExecutionMode.TEST: FullImageDataset(config, df, full_image_sample_transforms=transform) } # type: ignore execution_mode = ModelExecutionMode.TEST checkpoint_handler = get_default_checkpoint_handler( model_config=config, project_root=test_output_dirs.root_dir) # Mimic the behaviour that checkpoints are downloaded from blob storage into the checkpoints folder. create_model_and_store_checkpoint( config, config.checkpoint_folder / LAST_CHECKPOINT_FILE_NAME_WITH_SUFFIX) checkpoint_handler.additional_training_done() inference_results = model_testing.segmentation_model_test( config, execution_mode=execution_mode, checkpoint_paths=checkpoint_handler.get_checkpoints_to_test()) epoch_dir = config.outputs_folder / get_best_epoch_results_path( execution_mode) total_num_patients_column_name = f"total_{MetricsFileColumns.Patient.value}".lower( ) if not total_num_patients_column_name.endswith("s"): total_num_patients_column_name += "s" if use_partial_ground_truth: num_subjects = len(pd.unique(df["subject"])) if allow_partial_ground_truth: assert csv_column_contains_value( csv_file_path=epoch_dir / METRICS_AGGREGATES_FILE, column_name=total_num_patients_column_name, value=num_subjects, contains_only_value=True) assert csv_column_contains_value( csv_file_path=epoch_dir / SUBJECT_METRICS_FILE_NAME, column_name=MetricsFileColumns.Dice.value, value='', contains_only_value=False) else: aggregates_df = pd.read_csv(epoch_dir / METRICS_AGGREGATES_FILE) assert total_num_patients_column_name not in aggregates_df.columns # Only added if using partial ground truth assert not csv_column_contains_value( csv_file_path=epoch_dir / SUBJECT_METRICS_FILE_NAME, column_name=MetricsFileColumns.Dice.value, value='', contains_only_value=False) assert inference_results.metrics == pytest.approx(0.66606902, abs=1e-6) assert config.outputs_folder.is_dir() assert epoch_dir.is_dir() patient1 = io_util.load_nifti_image(train_and_test_data_dir / "id1_channel1.nii.gz") patient2 = io_util.load_nifti_image(train_and_test_data_dir / "id2_channel1.nii.gz") assert_file_contains_string(epoch_dir / DATASET_ID_FILE, placeholder_dataset_id) assert_file_contains_string(epoch_dir / GROUND_TRUTH_IDS_FILE, "region") assert_text_files_match( epoch_dir / model_testing.SUBJECT_METRICS_FILE_NAME, train_and_test_data_dir / model_testing.SUBJECT_METRICS_FILE_NAME) assert_text_files_match( epoch_dir / model_testing.METRICS_AGGREGATES_FILE, train_and_test_data_dir / model_testing.METRICS_AGGREGATES_FILE) # Plotting results vary between platforms. Can only check if the file is generated, but not its contents. assert (epoch_dir / model_testing.BOXPLOT_FILE).exists() assert_nifti_content(epoch_dir / "001" / "posterior_region.nii.gz", get_image_shape(patient1), patient1.header, [137], np.ubyte) assert_nifti_content(epoch_dir / "002" / "posterior_region.nii.gz", get_image_shape(patient2), patient2.header, [137], np.ubyte) assert_nifti_content(epoch_dir / "001" / DEFAULT_RESULT_IMAGE_NAME, get_image_shape(patient1), patient1.header, [1], np.ubyte) assert_nifti_content(epoch_dir / "002" / DEFAULT_RESULT_IMAGE_NAME, get_image_shape(patient2), patient2.header, [1], np.ubyte) assert_nifti_content(epoch_dir / "001" / "posterior_background.nii.gz", get_image_shape(patient1), patient1.header, [117], np.ubyte) assert_nifti_content(epoch_dir / "002" / "posterior_background.nii.gz", get_image_shape(patient2), patient2.header, [117], np.ubyte) thumbnails_folder = epoch_dir / model_testing.THUMBNAILS_FOLDER assert thumbnails_folder.is_dir() png_files = list(thumbnails_folder.glob("*.png")) overlays = [f for f in png_files if "_region_slice_" in str(f)] assert len(overlays) == len(df.subject.unique( )), "There should be one overlay/contour file per subject" # Writing dataset.csv normally happens at the beginning of training, # but this test reads off a saved checkpoint file. # Dataset.csv must be present for plot_cross_validation. config.write_dataset_files() # Test if the metrics files can be picked up correctly by the cross validation code config_and_files = get_config_and_results_for_offline_runs(config) result_files = config_and_files.files assert len(result_files) == 1 for file in result_files: assert file.execution_mode == execution_mode assert file.dataset_csv_file is not None assert file.dataset_csv_file.exists() assert file.metrics_file is not None assert file.metrics_file.exists()