def try_compare_scores_against_baselines(self, model_proc: ModelProcessing) -> None: """ Attempt comparison of scores against baseline scores and scatterplot creation if possible. """ if not isinstance(self.model_config, SegmentationModelBase): # keep type checker happy return from InnerEye.ML.baselines_util import compare_scores_against_baselines with logging_section("Comparing scores against baselines"): compare_scores_against_baselines(self.model_config, self.azure_config, model_proc)
def test_compare_scores_against_baselines_throws(model_proc_split_infer: Tuple[ ModelProcessing, int, Optional[bool]], test_output_dirs: OutputFolderForTests, caplog: Any) -> None: """ Test that exceptions are raised if the files necessary for baseline comparison are missing, then test that when all required files are present that baseline comparison files are written. :param model_proc: Model processing to test. :param test_output_dirs: Test output directories. :param caplog: Pytest capture logger. :return: None. """ (model_proc, number_of_cross_validation_splits, inference_on_test_set) = model_proc_split_infer config = SegmentationModelBase( should_validate=False, comparison_blob_storage_paths=[ ('Single', 'dummy_blob_single/outputs/epoch_120/Test'), ('5fold', 'dummy_blob_ensemble/outputs/epoch_120/Test') ], number_of_cross_validation_splits=number_of_cross_validation_splits) config.set_output_to(test_output_dirs.root_dir) azure_config = get_default_azure_config() # Disable inference config.inference_on_test_set = False config.ensemble_inference_on_test_set = False with caplog.at_level(logging.INFO): # With no inference output, test that a warning is written to logging compare_scores_against_baselines(model_config=config, azure_config=azure_config, model_proc=model_proc) assert INFERENCE_DISABLED_WARNING in caplog.text # Reset inference config.inference_on_test_set = inference_on_test_set config.ensemble_inference_on_test_set = None # If the BEST_EPOCH_FOLDER_NAME folder is missing, expect an exception to be raised. with pytest.raises(FileNotFoundError) as ex: compare_scores_against_baselines(model_config=config, azure_config=azure_config, model_proc=model_proc) assert "Cannot compare scores against baselines: no best epoch results found at" in str( ex) best_epoch_folder_path = config.outputs_folder if model_proc == ModelProcessing.ENSEMBLE_CREATION: best_epoch_folder_path = best_epoch_folder_path / OTHER_RUNS_SUBDIR_NAME / ENSEMBLE_SPLIT_NAME best_epoch_folder_path = best_epoch_folder_path / BEST_EPOCH_FOLDER_NAME / ModelExecutionMode.TEST.value best_epoch_folder_path.mkdir(parents=True) # If the BEST_EPOCH_FOLDER_NAME folder exists but DATASET_CSV_FILE_NAME is missing, # expect an exception to be raised. with pytest.raises(FileNotFoundError) as ex: compare_scores_against_baselines(model_config=config, azure_config=azure_config, model_proc=model_proc) assert "Not comparing with baselines because no " in str(ex) assert DATASET_CSV_FILE_NAME in str(ex) model_dataset_path = best_epoch_folder_path / DATASET_CSV_FILE_NAME dataset_df = create_dataset_df() dataset_df.to_csv(model_dataset_path) # If the BEST_EPOCH_FOLDER_NAME folder exists but SUBJECT_METRICS_FILE_NAME is missing, # expect an exception to be raised. with pytest.raises(FileNotFoundError) as ex: compare_scores_against_baselines(model_config=config, azure_config=azure_config, model_proc=model_proc) assert "Not comparing with baselines because no " in str(ex) assert SUBJECT_METRICS_FILE_NAME in str(ex) model_metrics_path = best_epoch_folder_path / SUBJECT_METRICS_FILE_NAME metrics_df = create_metrics_df() metrics_df.to_csv(model_metrics_path) baseline = create_comparison_baseline(dataset_df) # Patch get_comparison_baselines to return the baseline above. with mock.patch('InnerEye.ML.baselines_util.get_comparison_baselines', return_value=[baseline]): compare_scores_against_baselines(model_config=config, azure_config=azure_config, model_proc=model_proc) # Check the wilcoxoon results file is present and has expected contents. wilcoxon_path = best_epoch_folder_path / BASELINE_WILCOXON_RESULTS_FILE assert wilcoxon_path.is_file() wilcoxon_lines = [ line.strip() for line in wilcoxon_path.read_text().splitlines() ] check_wilcoxon_lines(wilcoxon_lines, baseline) # Check the full metrics results file is present. full_metrics_path = best_epoch_folder_path / FULL_METRICS_DATAFRAME_FILE assert full_metrics_path.is_file()