def generate_report(config: DeepLearningConfig, best_epoch: int, model_proc: ModelProcessing) -> None: logging.info("Saving report in html") if config.model_category not in [ModelCategory.Segmentation, ModelCategory.Classification]: return try: def get_epoch_path(mode: ModelExecutionMode) -> Path: p = get_epoch_results_path(best_epoch, mode=mode, model_proc=model_proc) return config.outputs_folder / p / METRICS_FILE_NAME path_to_best_epoch_train = get_epoch_path(ModelExecutionMode.TRAIN) path_to_best_epoch_val = get_epoch_path(ModelExecutionMode.VAL) path_to_best_epoch_test = get_epoch_path(ModelExecutionMode.TEST) output_dir = config.outputs_folder / OTHER_RUNS_SUBDIR_NAME / ENSEMBLE_SPLIT_NAME \ if model_proc == ModelProcessing.ENSEMBLE_CREATION else config.outputs_folder if config.model_category == ModelCategory.Segmentation: generate_segmentation_notebook(result_notebook=output_dir / REPORT_IPYNB, train_metrics=path_to_best_epoch_train, val_metrics=path_to_best_epoch_val, test_metrics=path_to_best_epoch_test) else: if isinstance(config, ScalarModelBase): generate_classification_notebook(result_notebook=output_dir / REPORT_IPYNB, train_metrics=path_to_best_epoch_train, val_metrics=path_to_best_epoch_val, test_metrics=path_to_best_epoch_test, dataset_csv_path=config.local_dataset / DATASET_CSV_FILE_NAME if config.local_dataset else None, dataset_subject_column=config.subject_column, dataset_file_column=config.image_file_column) else: logging.info(f"Cannot create report for config of type {type(config)}.") except Exception as ex: print_exception(ex, "Failed to generated reporting notebook.")
def generate_report(config: DeepLearningConfig, best_epoch: int, model_proc: ModelProcessing) -> None: logging.info("Saving report in html") if not config.is_segmentation_model: return try: def get_epoch_path(mode: ModelExecutionMode) -> Path: p = get_epoch_results_path(best_epoch, mode=mode, model_proc=model_proc) return config.outputs_folder / p / METRICS_FILE_NAME path_to_best_epoch_train = get_epoch_path(ModelExecutionMode.TRAIN) path_to_best_epoch_val = get_epoch_path(ModelExecutionMode.VAL) path_to_best_epoch_test = get_epoch_path(ModelExecutionMode.TEST) output_dir = config.outputs_folder / OTHER_RUNS_SUBDIR_NAME / ENSEMBLE_SPLIT_NAME \ if model_proc == ModelProcessing.ENSEMBLE_CREATION else config.outputs_folder generate_segmentation_notebook( result_notebook=output_dir / REPORT_IPYNB, train_metrics=path_to_best_epoch_train, val_metrics=path_to_best_epoch_val, test_metrics=path_to_best_epoch_test) except Exception as ex: print_exception(ex, "Failed to generated reporting notebook.")
def generate_report(self, model_proc: ModelProcessing) -> None: config = self.model_config if config.model_category not in [ModelCategory.Segmentation, ModelCategory.Classification]: logging.info(f"No reporting available for a model with category {config.model_category}") return logging.info("Saving report in HTML") try: def get_epoch_path(mode: ModelExecutionMode) -> Path: p = get_epoch_results_path(mode=mode, model_proc=model_proc) return config.outputs_folder / p / SUBJECT_METRICS_FILE_NAME path_to_best_epoch_train = get_epoch_path(ModelExecutionMode.TRAIN) path_to_best_epoch_val = get_epoch_path(ModelExecutionMode.VAL) path_to_best_epoch_test = get_epoch_path(ModelExecutionMode.TEST) output_dir = config.outputs_folder / OTHER_RUNS_SUBDIR_NAME / ENSEMBLE_SPLIT_NAME \ if model_proc == ModelProcessing.ENSEMBLE_CREATION else config.outputs_folder reports_dir = output_dir / reports_folder if not reports_dir.exists(): reports_dir.mkdir(exist_ok=False) if config.model_category == ModelCategory.Segmentation: generate_segmentation_notebook( result_notebook=reports_dir / get_ipynb_report_name(config.model_category.value), train_metrics=path_to_best_epoch_train, val_metrics=path_to_best_epoch_val, test_metrics=path_to_best_epoch_test) else: if isinstance(config, ScalarModelBase) and not isinstance(config, SequenceModelBase): generate_classification_notebook( result_notebook=reports_dir / get_ipynb_report_name(config.model_category.value), config=config, train_metrics=path_to_best_epoch_train, val_metrics=path_to_best_epoch_val, test_metrics=path_to_best_epoch_test) if len(config.class_names) > 1: generate_classification_multilabel_notebook( result_notebook=reports_dir / get_ipynb_report_name(f"{config.model_category.value}_multilabel"), config=config, train_metrics=path_to_best_epoch_train, val_metrics=path_to_best_epoch_val, test_metrics=path_to_best_epoch_test) else: logging.info(f"Cannot create report for config of type {type(config)}.") except Exception as ex: print_exception(ex, "Failed to generated reporting notebook.") raise
def _test_generate_segmentation_report_without_partial_ground_truth( test_output_dirs: OutputFolderForTests, metrics_file: Path) -> None: current_dir = test_output_dirs.make_sub_dir("test_segmentation_report") result_file = current_dir / "report.ipynb" result_html = generate_segmentation_notebook(result_notebook=result_file, test_metrics=metrics_file) assert result_file.is_file() assert result_html.is_file() assert result_html.suffix == ".html" # Check html contains the name of a key structure contents = result_html.read_text(encoding='utf-8') assert 'parotid_r' in contents
def test_generate_segmentation_report(test_output_dirs: OutputFolderForTests) -> None: reports_folder = Path(__file__).parent metrics_file = reports_folder / "metrics_hn.csv" current_dir = test_output_dirs.make_sub_dir("test_segmentation_report") result_file = current_dir / "report.ipynb" result_html = generate_segmentation_notebook(result_notebook=result_file, test_metrics=metrics_file) assert result_file.is_file() assert result_html.is_file() assert result_html.suffix == ".html" # Check html contains the name of a key structure contents = result_html.read_text(encoding='utf-8') assert 'parotid_r' in contents
def _test_generate_segmentation_report_with_partial_ground_truth( test_output_dirs: OutputFolderForTests, original_metrics_file: Path) -> None: """ The test without partial ground truth should cover more detail, here we just check that providing partial ground truth results in some labels having a lower user count. """ original_metrics = pd.read_csv(original_metrics_file) partial_metrics = original_metrics partial_metrics.loc[ partial_metrics['Structure'].eq('brainstem') & partial_metrics['Patient'].isin([14, 15, 19]), ['Dice', 'HausdorffDistance_mm', 'MeanDistance_mm']] = NaN current_dir = test_output_dirs.make_sub_dir("test_segmentation_report") partial_metrics_file = current_dir / "metrics_hn.csv" result_file = current_dir / "report.ipynb" partial_metrics.to_csv(partial_metrics_file, index=False, float_format="%.3f", na_rep="") result_html = generate_segmentation_notebook( result_notebook=result_file, test_metrics=partial_metrics_file) result_html_text = result_html.read_text(encoding='utf-8') # Look for this row in the HTML Dice table: # <td>brainstem</td>\n <td>0.82600</td>\n <td>0.8570</td>\n <td>0.87600</td>\n <td>17.0</td>\n # It shows that for the brainstem label there are only 17, not 20, patients with that label, # because we removed the brainstem label for patients 14, 15, and 19. def get_patient_count_for_structure(structure: str, text: str) -> float: regex = f"<td>{structure}" + r"<\/td>(\n\s*<td>[0-9\.]*<\/td>){3}\n\s*<td>([0-9\.]*)" # which results in, for example, this regex: # regex = "<td>brainstem<\/td>(\n\s*<td>[0-9\.]*<\/td>){3}\n\s*<td>([0-9\.]*)" match = re.search(regex, text) if not match: return NaN patient_count_as_string = match.group(2) return float(patient_count_as_string) num_patients_with_lacrimal_gland_l_label = get_patient_count_for_structure( "lacrimal_gland_l", result_html_text) num_patients_with_brainstem_label = get_patient_count_for_structure( "brainstem", result_html_text) assert num_patients_with_lacrimal_gland_l_label == 20.0 assert num_patients_with_brainstem_label == 17.0