def generate_report(config: DeepLearningConfig, best_epoch: int, model_proc: ModelProcessing) -> None:
logging.info("Saving report in html")
if config.model_category not in [ModelCategory.Segmentation, ModelCategory.Classification]:
return
try:
def get_epoch_path(mode: ModelExecutionMode) -> Path:
p = get_epoch_results_path(best_epoch, mode=mode, model_proc=model_proc)
return config.outputs_folder / p / METRICS_FILE_NAME
path_to_best_epoch_train = get_epoch_path(ModelExecutionMode.TRAIN)
path_to_best_epoch_val = get_epoch_path(ModelExecutionMode.VAL)
path_to_best_epoch_test = get_epoch_path(ModelExecutionMode.TEST)
output_dir = config.outputs_folder / OTHER_RUNS_SUBDIR_NAME / ENSEMBLE_SPLIT_NAME \
if model_proc == ModelProcessing.ENSEMBLE_CREATION else config.outputs_folder
if config.model_category == ModelCategory.Segmentation:
generate_segmentation_notebook(result_notebook=output_dir / REPORT_IPYNB,
train_metrics=path_to_best_epoch_train,
val_metrics=path_to_best_epoch_val,
test_metrics=path_to_best_epoch_test)
else:
if isinstance(config, ScalarModelBase):
generate_classification_notebook(result_notebook=output_dir / REPORT_IPYNB,
train_metrics=path_to_best_epoch_train,
val_metrics=path_to_best_epoch_val,
test_metrics=path_to_best_epoch_test,
dataset_csv_path=config.local_dataset / DATASET_CSV_FILE_NAME
if config.local_dataset else None,
dataset_subject_column=config.subject_column,
dataset_file_column=config.image_file_column)
else:
logging.info(f"Cannot create report for config of type {type(config)}.")
except Exception as ex:
print_exception(ex, "Failed to generated reporting notebook.")
def generate_report(config: DeepLearningConfig, best_epoch: int,
model_proc: ModelProcessing) -> None:
logging.info("Saving report in html")
if not config.is_segmentation_model:
return
try:
def get_epoch_path(mode: ModelExecutionMode) -> Path:
p = get_epoch_results_path(best_epoch,
mode=mode,
model_proc=model_proc)
return config.outputs_folder / p / METRICS_FILE_NAME
path_to_best_epoch_train = get_epoch_path(ModelExecutionMode.TRAIN)
path_to_best_epoch_val = get_epoch_path(ModelExecutionMode.VAL)
path_to_best_epoch_test = get_epoch_path(ModelExecutionMode.TEST)
output_dir = config.outputs_folder / OTHER_RUNS_SUBDIR_NAME / ENSEMBLE_SPLIT_NAME \
if model_proc == ModelProcessing.ENSEMBLE_CREATION else config.outputs_folder
generate_segmentation_notebook(
result_notebook=output_dir / REPORT_IPYNB,
train_metrics=path_to_best_epoch_train,
val_metrics=path_to_best_epoch_val,
test_metrics=path_to_best_epoch_test)
except Exception as ex:
print_exception(ex, "Failed to generated reporting notebook.")
def generate_report(self, model_proc: ModelProcessing) -> None:
config = self.model_config
if config.model_category not in [ModelCategory.Segmentation, ModelCategory.Classification]:
logging.info(f"No reporting available for a model with category {config.model_category}")
return
logging.info("Saving report in HTML")
try:
def get_epoch_path(mode: ModelExecutionMode) -> Path:
p = get_epoch_results_path(mode=mode, model_proc=model_proc)
return config.outputs_folder / p / SUBJECT_METRICS_FILE_NAME
path_to_best_epoch_train = get_epoch_path(ModelExecutionMode.TRAIN)
path_to_best_epoch_val = get_epoch_path(ModelExecutionMode.VAL)
path_to_best_epoch_test = get_epoch_path(ModelExecutionMode.TEST)
output_dir = config.outputs_folder / OTHER_RUNS_SUBDIR_NAME / ENSEMBLE_SPLIT_NAME \
if model_proc == ModelProcessing.ENSEMBLE_CREATION else config.outputs_folder
reports_dir = output_dir / reports_folder
if not reports_dir.exists():
reports_dir.mkdir(exist_ok=False)
if config.model_category == ModelCategory.Segmentation:
generate_segmentation_notebook(
result_notebook=reports_dir / get_ipynb_report_name(config.model_category.value),
train_metrics=path_to_best_epoch_train,
val_metrics=path_to_best_epoch_val,
test_metrics=path_to_best_epoch_test)
else:
if isinstance(config, ScalarModelBase) and not isinstance(config, SequenceModelBase):
generate_classification_notebook(
result_notebook=reports_dir / get_ipynb_report_name(config.model_category.value),
config=config,
train_metrics=path_to_best_epoch_train,
val_metrics=path_to_best_epoch_val,
test_metrics=path_to_best_epoch_test)
if len(config.class_names) > 1:
generate_classification_multilabel_notebook(
result_notebook=reports_dir / get_ipynb_report_name(f"{config.model_category.value}_multilabel"),
config=config,
train_metrics=path_to_best_epoch_train,
val_metrics=path_to_best_epoch_val,
test_metrics=path_to_best_epoch_test)
else:
logging.info(f"Cannot create report for config of type {type(config)}.")
except Exception as ex:
print_exception(ex, "Failed to generated reporting notebook.")
raise
def _test_generate_segmentation_report_without_partial_ground_truth(
test_output_dirs: OutputFolderForTests, metrics_file: Path) -> None:
current_dir = test_output_dirs.make_sub_dir("test_segmentation_report")
result_file = current_dir / "report.ipynb"
result_html = generate_segmentation_notebook(result_notebook=result_file,
test_metrics=metrics_file)
assert result_file.is_file()
assert result_html.is_file()
assert result_html.suffix == ".html"
# Check html contains the name of a key structure
contents = result_html.read_text(encoding='utf-8')
assert 'parotid_r' in contents
def test_generate_segmentation_report(test_output_dirs: OutputFolderForTests) -> None:
reports_folder = Path(__file__).parent
metrics_file = reports_folder / "metrics_hn.csv"
current_dir = test_output_dirs.make_sub_dir("test_segmentation_report")
result_file = current_dir / "report.ipynb"
result_html = generate_segmentation_notebook(result_notebook=result_file,
test_metrics=metrics_file)
assert result_file.is_file()
assert result_html.is_file()
assert result_html.suffix == ".html"
# Check html contains the name of a key structure
contents = result_html.read_text(encoding='utf-8')
assert 'parotid_r' in contents
def _test_generate_segmentation_report_with_partial_ground_truth(
test_output_dirs: OutputFolderForTests,
original_metrics_file: Path) -> None:
"""
The test without partial ground truth should cover more detail, here we just check that providing
partial ground truth results in some labels having a lower user count.
"""
original_metrics = pd.read_csv(original_metrics_file)
partial_metrics = original_metrics
partial_metrics.loc[
partial_metrics['Structure'].eq('brainstem')
& partial_metrics['Patient'].isin([14, 15, 19]),
['Dice', 'HausdorffDistance_mm', 'MeanDistance_mm']] = NaN
current_dir = test_output_dirs.make_sub_dir("test_segmentation_report")
partial_metrics_file = current_dir / "metrics_hn.csv"
result_file = current_dir / "report.ipynb"
partial_metrics.to_csv(partial_metrics_file,
index=False,
float_format="%.3f",
na_rep="")
result_html = generate_segmentation_notebook(
result_notebook=result_file, test_metrics=partial_metrics_file)
result_html_text = result_html.read_text(encoding='utf-8')
# Look for this row in the HTML Dice table:
# <td>brainstem</td>\n <td>0.82600</td>\n <td>0.8570</td>\n <td>0.87600</td>\n <td>17.0</td>\n
# It shows that for the brainstem label there are only 17, not 20, patients with that label,
# because we removed the brainstem label for patients 14, 15, and 19.
def get_patient_count_for_structure(structure: str, text: str) -> float:
regex = f"<td>{structure}" + r"<\/td>(\n\s*<td>[0-9\.]*<\/td>){3}\n\s*<td>([0-9\.]*)"
# which results in, for example, this regex:
# regex = "<td>brainstem<\/td>(\n\s*<td>[0-9\.]*<\/td>){3}\n\s*<td>([0-9\.]*)"
match = re.search(regex, text)
if not match:
return NaN
patient_count_as_string = match.group(2)
return float(patient_count_as_string)
num_patients_with_lacrimal_gland_l_label = get_patient_count_for_structure(
"lacrimal_gland_l", result_html_text)
num_patients_with_brainstem_label = get_patient_count_for_structure(
"brainstem", result_html_text)
assert num_patients_with_lacrimal_gland_l_label == 20.0
assert num_patients_with_brainstem_label == 17.0