def plot_pr_and_roc_curves_from_csv(
metrics_csv: Path,
config: ScalarModelBase,
data_split: Optional[ModelExecutionMode] = None,
is_crossval_report: bool = False) -> None:
"""
Given the CSV written during inference time and the model config, plot the ROC and PR curves for all prediction
targets.
:param metrics_csv: Path to the metrics CSV file.
:param config: Model config.
:param data_split: Whether to filter the CSV file for Train, Val, or Test results (default: no filtering).
:param is_crossval_report: If True, assumes CSV contains results for multiple cross-validation runs and plots the
curves with median and confidence intervals. Otherwise, plots curves for a single run.
"""
for prediction_target in config.target_names:
print_header(f"Class: {prediction_target}", level=3)
if is_crossval_report:
all_metrics = [
get_labels_and_predictions(metrics_csv,
prediction_target,
crossval_split_index=crossval_split,
data_split=data_split)
for crossval_split in range(
config.number_of_cross_validation_splits)
]
plot_pr_and_roc_curves_crossval(all_metrics)
else:
metrics = get_labels_and_predictions(metrics_csv,
prediction_target,
data_split=data_split)
plot_pr_and_roc_curves(metrics)
def display_metric(df: pd.DataFrame, metric_name: str, outlier_range: float,
max_row_count: int, high_values_are_good: bool) -> None:
"""
Displays a dataframe with a metric per structure, first showing the
:param max_row_count: The number of rows to print when showing the lowest score patients
:param df: The dataframe with metrics per structure
:param metric_name: The metric to sort by.
:param outlier_range: The standard deviation range data points must fall outside of to be considered an outlier
:param high_values_are_good: If true, high values of the metric indicate good performance. If false, low
values indicate good performance.
"""
print_header(metric_name, level=2)
# Display with best structures first.
display_without_index(
describe_score(df, metric_name, ascending=(not high_values_are_good)))
print_header(f"{max_row_count} worst patients by {metric_name}", level=3)
print(
f"Rows are marked as outliers if {metric_name} is more than {outlier_range} standard deviations from "
f"the mean.")
display_without_index(
worst_patients_and_outliers(df,
outlier_range,
metric_name,
high_values_are_good,
max_row_count=max_row_count))
boxplot_per_structure(df, column_name=metric_name, title=metric_name)
plt.show()
def plot_image_from_filepath(filepath: Path, im_size: Tuple) -> bool:
"""
Plots a 2D image given the filepath. Returns false if the image could not be plotted (for example, if it was 3D).
:param filepath: Path to image
:param im_size: Display size for image
:return: True if image was plotted, False otherwise
"""
image = load_image_in_known_formats(filepath,
load_segmentation=False).images
if not image.ndim == 2 and not (image.ndim == 3 and image.shape[0] == 1):
print_header(f"Image has unsupported shape {image.shape}", level=0)
return False
if image.ndim == 3 and image.shape[0] == 1:
image = image.squeeze(0)
# normalize to make sure pixels are plottable
min = image.min()
max = image.max()
image = (image - min) / (max - min)
image = image * 255.
image = np.repeat(image[:, :, None], 3, 2)
image = image.astype(np.uint8)
display(Image.fromarray(image).resize(im_size))
return True
def plot_pr_and_roc_curves_from_csv(metrics_csv: Path, config: ScalarModelBase) -> None:
"""
Given the csv written during inference time and the model config,
plot the ROC and PR curves for all prediction targets.
"""
for prediction_target in config.class_names:
print_header(f"Class {prediction_target}", level=3)
metrics = get_labels_and_predictions(metrics_csv, prediction_target)
plot_pr_and_roc_curves(metrics)
def plot_pr_and_roc_curves(labels_and_model_outputs: LabelsAndPredictions) -> None:
"""
Given a LabelsAndPredictions object, plot the ROC and PR curves.
"""
print_header("ROC and PR curves", level=3)
_, ax = plt.subplots(1, 2)
fpr, tpr, thresholds = roc_curve(labels_and_model_outputs.labels, labels_and_model_outputs.model_outputs)
plot_auc(fpr, tpr, "ROC Curve", ax[0])
precision, recall, thresholds = precision_recall_curve(labels_and_model_outputs.labels,
labels_and_model_outputs.model_outputs)
plot_auc(recall, precision, "PR Curve", ax[1])
plt.show()
def plot_pr_and_roc_curves_from_csv(metrics_csv: Path) -> None:
"""
Given a csv file, read the predicted values and ground truth labels and plot the ROC and PR curves.
"""
print_header("ROC and PR curves", level=3)
results = get_results(metrics_csv)
_, ax = plt.subplots(1, 2)
fpr, tpr, thresholds = roc_curve(results.labels, results.model_outputs)
plot_auc(fpr, tpr, "ROC Curve", ax[0])
precision, recall, thresholds = precision_recall_curve(
results.labels, results.model_outputs)
plot_auc(recall, precision, "PR Curve", ax[1])
plt.show()
def display_metric(df: pd.DataFrame, metric_name: str,
high_values_are_good: bool) -> None:
"""
Displays a dataframe with a metric per structure, first showing the
:param df: The dataframe with metrics per structure
:param metric_name: The metric to sort by.
:param high_values_are_good: If true, high values of the metric indicate good performance. If false, low
values indicate good performance.
"""
print_header(metric_name, level=2)
# Display with best structures first.
display(
describe_score(df, metric_name, ascending=(not high_values_are_good)))
print_header(f"Worst patients by {metric_name}", level=3)
display(worst_patients(df, metric_name, ascending=high_values_are_good))
boxplot_per_structure(df, column_name=metric_name, title=metric_name)
plt.show()
def print_metrics_for_thresholded_output_for_all_prediction_targets(
val_metrics_csv: Path, test_metrics_csv: Path,
config: ScalarModelBase) -> None:
"""
Given csvs written during inference for the validation and test sets, print out metrics for every combination of
prediction targets that exist in the dataset (i.e. for every subset of classes that occur in the dataset).
:param val_metrics_csv: Csv written during inference time for the val set. This is used to determine the
optimal threshold for classification.
:param test_metrics_csv: Csv written during inference time for the test set. Metrics are calculated for this csv.
:param config: Model config
"""
unique_prediction_target_combinations = get_unique_prediction_target_combinations(
config)
all_prediction_target_combinations = list(
set([
frozenset([prediction_target])
for prediction_target in config.class_names
])
| unique_prediction_target_combinations)
thresholds_per_prediction_target = []
for label in config.class_names:
val_metrics = get_labels_and_predictions(val_metrics_csv, label)
test_metrics = get_labels_and_predictions(test_metrics_csv, label)
thresholds_per_prediction_target.append(
get_metric(val_labels_and_predictions=val_metrics,
test_labels_and_predictions=test_metrics,
metric=ReportedMetrics.OptimalThreshold))
for labels in all_prediction_target_combinations:
print_header(f"Class {'|'.join(labels) or 'Negative'}", level=3)
val_metrics = get_labels_and_predictions_for_prediction_target_set(
csv=val_metrics_csv,
prediction_target_set_to_match=list(labels),
all_prediction_targets=config.class_names,
thresholds_per_prediction_target=thresholds_per_prediction_target)
test_metrics = get_labels_and_predictions_for_prediction_target_set(
csv=test_metrics_csv,
prediction_target_set_to_match=list(labels),
all_prediction_targets=config.class_names,
thresholds_per_prediction_target=thresholds_per_prediction_target)
print_metrics(val_labels_and_predictions=val_metrics,
test_labels_and_predictions=test_metrics,
is_thresholded=True)
def display_metric(df: pd.DataFrame, metric_name: str, outlier_range: float,
high_values_are_good: bool) -> None:
"""
Displays a dataframe with a metric per structure, first showing the
:param df: The dataframe with metrics per structure
:param metric_name: The metric to sort by.
:param outlier_range: The standard deviation range data points must fall outside of to be considered an outlier
:param high_values_are_good: If true, high values of the metric indicate good performance. If false, low
values indicate good performance.
"""
print_header(metric_name, level=2)
# Display with best structures first.
display(
describe_score(df, metric_name, ascending=(not high_values_are_good)))
print_header(f"Worst patients by {metric_name} (< {outlier_range} sd)",
level=3)
display(
display_outliers(df, outlier_range, metric_name, high_values_are_good))
boxplot_per_structure(df, column_name=metric_name, title=metric_name)
plt.show()
def print_metrics_for_all_prediction_targets(val_metrics_csv: Path,
test_metrics_csv: Path,
config: ScalarModelBase,
is_thresholded: bool = False) -> None:
"""
Given csvs written during inference for the validation and test sets, print out metrics for every prediction target
in the config.
:param val_metrics_csv: Csv written during inference time for the val set. This is used to determine the
optimal threshold for classification.
:param test_metrics_csv: Csv written during inference time for the test set. Metrics are calculated for this csv.
:param config: Model config
:param is_thresholded: Whether the model outputs are binary (they have been thresholded at some point)
or are floating point numbers.
"""
for prediction_target in config.class_names:
print_header(f"Class {prediction_target}", level=3)
val_metrics = get_labels_and_predictions(val_metrics_csv, prediction_target)
test_metrics = get_labels_and_predictions(test_metrics_csv, prediction_target)
print_metrics(val_labels_and_predictions=val_metrics, test_labels_and_predictions=test_metrics, is_thresholded=is_thresholded)
def plot_image_for_subject(subject_id: str, dataset_df: pd.DataFrame,
dataset_subject_column: str,
dataset_file_column: str, dataset_dir: Path,
im_size: Tuple, model_output: float,
header: Optional[str]) -> None:
"""
Given a subject ID, plots the corresponding image.
:param subject_id: Subject to plot image for
:param dataset_df: Dataset dataframe (from the datset.csv file)
:param dataset_subject_column: Name of the column with the subject IDs
:param dataset_file_column: Name of the column with the image filepaths
:param dataset_dir: Path to the dataset
:param im_size: Display size for image
:param model_output: The predicted value for this image
:param header: Optional header printed along with the subject ID and score for the image.
"""
print_header("", level=4)
if header:
print_header(header, level=4)
print_header(f"ID: {subject_id} Score: {model_output}", level=4)
filepath = get_image_filepath_from_subject_id(
subject_id=str(subject_id),
dataset_df=dataset_df,
dataset_subject_column=dataset_subject_column,
dataset_file_column=dataset_file_column,
dataset_dir=dataset_dir)
if not filepath:
print_header(
f"Subject ID {subject_id} not found, or found duplicate entries for this subject "
f"in column {dataset_subject_column} in the csv file. "
f"Note: Reports with datasets that use channel columns in the dataset.csv "
f"are not yet supported.")
return
success = plot_image_from_filepath(filepath, im_size=im_size)
if not success:
print_header(
"Unable to plot image: image must be 2D with shape [w, h] or [1, w, h].",
level=0)
def print_metrics_for_all_prediction_targets(
csv_to_set_optimal_threshold: Path,
csv_to_compute_metrics: Path,
config: ScalarModelBase,
data_split_to_set_optimal_threshold: Optional[
ModelExecutionMode] = None,
data_split_to_compute_metrics: Optional[ModelExecutionMode] = None,
is_thresholded: bool = False,
is_crossval_report: bool = False) -> None:
"""
Given CSVs written during inference for the validation and test sets, print out metrics for every prediction target
in the config.
:param csv_to_set_optimal_threshold: CSV written during inference time for the val set. This is used to determine
the optimal threshold for classification.
:param csv_to_compute_metrics: CSV written during inference time for the test set. Metrics are calculated for
this CSV.
:param config: Model config
:param data_split_to_set_optimal_threshold: Whether to filter the validation CSV file for Train, Val, or Test
results (default: no filtering).
:param data_split_to_compute_metrics: Whether to filter the test CSV file for Train, Val, or Test results
(default: no filtering).
:param is_thresholded: Whether the model outputs are binary (they have been thresholded at some point)
or are floating point numbers.
:param is_crossval_report: If True, assumes CSVs contain results for multiple cross-validation runs and prints the
metrics along with means and standard deviations. Otherwise, prints metrics for a single run.
"""
for prediction_target in config.target_names:
print_header(f"Class: {prediction_target}", level=3)
rows, header = get_metrics_table_for_prediction_target(
csv_to_set_optimal_threshold=csv_to_set_optimal_threshold,
data_split_to_set_optimal_threshold=
data_split_to_set_optimal_threshold,
csv_to_compute_metrics=csv_to_compute_metrics,
data_split_to_compute_metrics=data_split_to_compute_metrics,
config=config,
prediction_target=prediction_target,
is_thresholded=is_thresholded,
is_crossval_report=is_crossval_report)
print_table(rows, header)
def plot_k_best_and_worst_performing(val_metrics_csv: Path,
test_metrics_csv: Path, k: int,
prediction_target: str,
config: ScalarModelBase) -> None:
"""
Plot images for the top "k" best predictions (i.e. correct classifications where the model was the most certain)
and the top "k" worst predictions (i.e. misclassifications where the model was the most confident).
:param val_metrics_csv: Path to one of the metrics csvs written during inference. This set of metrics will be
used to determine the thresholds for predicting labels on the test set. The best and worst
performing subjects will not be printed out for this csv.
:param test_metrics_csv: Path to one of the metrics csvs written during inference. This is the csv for which
best and worst performing subjects will be printed out.
:param k: Number of subjects of each category to print out.
:param prediction_target: The class label to filter on
:param config: scalar model config object
"""
results = get_k_best_and_worst_performing(
val_metrics_csv=val_metrics_csv,
test_metrics_csv=test_metrics_csv,
k=k,
prediction_target=prediction_target)
if results is None:
print_header("Empty validation or test set", level=4)
return
test_metrics = pd.read_csv(test_metrics_csv, dtype=str)
df = config.read_dataset_if_needed()
dataset = ScalarDataset(args=config, data_frame=df)
im_width = 800
print_header("", level=2)
print_header(f"Top {k} false positives", level=2)
for index, (subject, model_output) in enumerate(
zip(results.false_positives[LoggingColumns.Patient.value],
results.false_positives[LoggingColumns.ModelOutput.value])):
plot_image_for_subject(subject_id=str(subject),
dataset=dataset,
im_width=im_width,
model_output=model_output,
header="False Positive",
config=config,
metrics_df=test_metrics)
print_header(f"Top {k} false negatives", level=2)
for index, (subject, model_output) in enumerate(
zip(results.false_negatives[LoggingColumns.Patient.value],
results.false_negatives[LoggingColumns.ModelOutput.value])):
plot_image_for_subject(subject_id=str(subject),
dataset=dataset,
im_width=im_width,
model_output=model_output,
header="False Negative",
config=config,
metrics_df=test_metrics)
print_header(f"Top {k} true positives", level=2)
for index, (subject, model_output) in enumerate(
zip(results.true_positives[LoggingColumns.Patient.value],
results.true_positives[LoggingColumns.ModelOutput.value])):
plot_image_for_subject(subject_id=str(subject),
dataset=dataset,
im_width=im_width,
model_output=model_output,
header="True Positive",
config=config,
metrics_df=test_metrics)
print_header(f"Top {k} true negatives", level=2)
for index, (subject, model_output) in enumerate(
zip(results.true_negatives[LoggingColumns.Patient.value],
results.true_negatives[LoggingColumns.ModelOutput.value])):
plot_image_for_subject(subject_id=str(subject),
dataset=dataset,
im_width=im_width,
model_output=model_output,
header="True Negative",
config=config,
metrics_df=test_metrics)
def print_metrics(val_labels_and_predictions: LabelsAndPredictions,
test_labels_and_predictions: LabelsAndPredictions,
is_thresholded: bool = False) -> None:
"""
Given LabelsAndPredictions objects for the validation and test sets, print out some metrics.
:param val_labels_and_predictions: LabelsAndPredictions object for the val set. This is used to determine the
optimal threshold for classification.
:param test_labels_and_predictions: LabelsAndPredictions object for the test set. Metrics are calculated for this
set.
:param is_thresholded: Whether the model outputs are binary (they have been thresholded at some point)
or are floating point numbers.
:return:
"""
optimal_threshold = 0.5 if is_thresholded else None
if not is_thresholded:
roc_auc = get_metric(val_labels_and_predictions=val_labels_and_predictions,
test_labels_and_predictions=test_labels_and_predictions,
metric=ReportedMetrics.AUC_ROC)
print_header(f"Area under ROC Curve: {roc_auc:.4f}", level=4)
pr_auc = get_metric(val_labels_and_predictions=val_labels_and_predictions,
test_labels_and_predictions=test_labels_and_predictions,
metric=ReportedMetrics.AUC_PR)
print_header(f"Area under PR Curve: {pr_auc:.4f}", level=4)
optimal_threshold = get_metric(val_labels_and_predictions=val_labels_and_predictions,
test_labels_and_predictions=test_labels_and_predictions,
metric=ReportedMetrics.OptimalThreshold)
print_header(f"Optimal threshold: {optimal_threshold: .4f}", level=4)
accuracy = get_metric(val_labels_and_predictions=val_labels_and_predictions,
test_labels_and_predictions=test_labels_and_predictions,
metric=ReportedMetrics.Accuracy,
optimal_threshold=optimal_threshold)
print_header(f"Accuracy at optimal threshold: {accuracy:.4f}", level=4)
fpr = get_metric(val_labels_and_predictions=val_labels_and_predictions,
test_labels_and_predictions=test_labels_and_predictions,
metric=ReportedMetrics.FalsePositiveRate,
optimal_threshold=optimal_threshold)
print_header(f"Specificity at optimal threshold: {1 - fpr:.4f}", level=4)
fnr = get_metric(val_labels_and_predictions=val_labels_and_predictions,
test_labels_and_predictions=test_labels_and_predictions,
metric=ReportedMetrics.FalseNegativeRate,
optimal_threshold=optimal_threshold)
print_header(f"Sensitivity at optimal threshold: {1 - fnr:.4f}", level=4)
print_header("", level=4)
def plot_k_best_and_worst_performing(val_metrics_csv: Path,
test_metrics_csv: Path, k: int,
dataset_csv_path: Path,
dataset_subject_column: str,
dataset_file_column: str) -> None:
"""
Plot images for the top "k" best predictions (i.e. correct classifications where the model was the most certain)
and the top "k" worst predictions (i.e. misclassifications where the model was the most confident).
"""
results = get_k_best_and_worst_performing(
val_metrics_csv=val_metrics_csv,
test_metrics_csv=test_metrics_csv,
k=k)
dataset_df = pd.read_csv(dataset_csv_path)
dataset_dir = dataset_csv_path.parent
im_size = (800, 800)
print_header("", level=2)
print_header(f"Top {k} false positives", level=2)
for index, (subject, model_output) in enumerate(
zip(results.false_positives[LoggingColumns.Patient.value],
results.false_positives[LoggingColumns.ModelOutput.value])):
plot_image_for_subject(subject_id=str(subject),
dataset_df=dataset_df,
dataset_subject_column=dataset_subject_column,
dataset_file_column=dataset_file_column,
dataset_dir=dataset_dir,
im_size=im_size,
model_output=model_output,
header="False Positive")
print_header(f"Top {k} false negatives", level=2)
for index, (subject, model_output) in enumerate(
zip(results.false_negatives[LoggingColumns.Patient.value],
results.false_negatives[LoggingColumns.ModelOutput.value])):
plot_image_for_subject(subject_id=str(subject),
dataset_df=dataset_df,
dataset_subject_column=dataset_subject_column,
dataset_file_column=dataset_file_column,
dataset_dir=dataset_dir,
im_size=im_size,
model_output=model_output,
header="False Negative")
print_header(f"Top {k} true positives", level=2)
for index, (subject, model_output) in enumerate(
zip(results.true_positives[LoggingColumns.Patient.value],
results.true_positives[LoggingColumns.ModelOutput.value])):
plot_image_for_subject(subject_id=str(subject),
dataset_df=dataset_df,
dataset_subject_column=dataset_subject_column,
dataset_file_column=dataset_file_column,
dataset_dir=dataset_dir,
im_size=im_size,
model_output=model_output,
header="True Positive")
print_header(f"Top {k} true negatives", level=2)
for index, (subject, model_output) in enumerate(
zip(results.true_negatives[LoggingColumns.Patient.value],
results.true_negatives[LoggingColumns.ModelOutput.value])):
plot_image_for_subject(subject_id=str(subject),
dataset_df=dataset_df,
dataset_subject_column=dataset_subject_column,
dataset_file_column=dataset_file_column,
dataset_dir=dataset_dir,
im_size=im_size,
model_output=model_output,
header="True Negative")
def print_k_best_and_worst_performing(val_metrics_csv: Path,
test_metrics_csv: Path, k: int) -> None:
"""
Print the top "k" best predictions (i.e. correct classifications where the model was the most certain) and the
top "k" worst predictions (i.e. misclassifications where the model was the most confident).
"""
results = get_k_best_and_worst_performing(
val_metrics_csv=val_metrics_csv,
test_metrics_csv=test_metrics_csv,
k=k)
print_header(f"Top {k} false positives", level=2)
for index, (subject, model_output) in enumerate(
zip(results.false_positives[LoggingColumns.Patient.value],
results.false_positives[LoggingColumns.ModelOutput.value])):
print_header(f"{index+1}. ID {subject} Score: {model_output:.5f}",
level=4)
print_header(f"Top {k} false negatives", level=2)
for index, (subject, model_output) in enumerate(
zip(results.false_negatives[LoggingColumns.Patient.value],
results.false_negatives[LoggingColumns.ModelOutput.value])):
print_header(f"{index+1}. ID {subject} Score: {model_output:.5f}",
level=4)
print_header(f"Top {k} true positives", level=2)
for index, (subject, model_output) in enumerate(
zip(results.true_positives[LoggingColumns.Patient.value],
results.true_positives[LoggingColumns.ModelOutput.value])):
print_header(f"{index+1}. ID {subject} Score: {model_output:.5f}",
level=4)
print_header(f"Top {k} true negatives", level=2)
for index, (subject, model_output) in enumerate(
zip(results.true_negatives[LoggingColumns.Patient.value],
results.true_negatives[LoggingColumns.ModelOutput.value])):
print_header(f"{index+1}. ID {subject} Score: {model_output:.5f}",
level=4)
def print_metrics(val_metrics_csv: Path, test_metrics_csv: Path) -> None:
"""
Given a csv file, read the predicted values and ground truth labels and print out some metrics.
"""
roc_auc = get_metric(val_metrics_csv=val_metrics_csv,
test_metrics_csv=test_metrics_csv,
metric=ReportedMetrics.AUC_ROC)
print_header(f"Area under ROC Curve: {roc_auc:.4f}", level=4)
pr_auc = get_metric(val_metrics_csv=val_metrics_csv,
test_metrics_csv=test_metrics_csv,
metric=ReportedMetrics.AUC_PR)
print_header(f"Area under PR Curve: {pr_auc:.4f}", level=4)
optimal_threshold = get_metric(val_metrics_csv=val_metrics_csv,
test_metrics_csv=test_metrics_csv,
metric=ReportedMetrics.OptimalThreshold)
print_header(f"Optimal threshold: {optimal_threshold: .4f}", level=4)
accuracy = get_metric(val_metrics_csv=val_metrics_csv,
test_metrics_csv=test_metrics_csv,
metric=ReportedMetrics.Accuracy)
print_header(f"Accuracy at optimal threshold: {accuracy:.4f}", level=4)
fpr = get_metric(val_metrics_csv=val_metrics_csv,
test_metrics_csv=test_metrics_csv,
metric=ReportedMetrics.FalsePositiveRate)
print_header(f"Specificity at optimal threshold: {1-fpr:.4f}", level=4)
fnr = get_metric(val_metrics_csv=val_metrics_csv,
test_metrics_csv=test_metrics_csv,
metric=ReportedMetrics.FalseNegativeRate)
print_header(f"Sensitivity at optimal threshold: {1-fnr:.4f}", level=4)
def plot_image_for_subject(subject_id: str,
dataset: ScalarDataset,
im_width: int,
model_output: float,
header: Optional[str],
config: ScalarModelBase,
metrics_df: Optional[pd.DataFrame] = None) -> None:
"""
Given a subject ID, plots the corresponding image.
:param subject_id: Subject to plot image for
:param dataset: scalar dataset object
:param im_width: Display width for image
:param model_output: The predicted value for this image
:param header: Optional header printed along with the subject ID and score for the image.
:param config: model config
:param metrics_df: dataframe with the metrics written out during inference time
"""
print_header("", level=4)
if header:
print_header(header, level=4)
labels = get_image_labels_from_subject_id(subject_id=subject_id,
dataset=dataset,
config=config)
print_header(f"True labels: {', '.join(labels) if labels else 'Negative'}",
level=4)
if metrics_df is not None:
all_model_outputs = get_image_outputs_from_subject_id(
subject_id=subject_id, metrics_df=metrics_df)
print_header(f"ID: {subject_id}", level=4)
print_header(
f"Model output: {', '.join([':'.join([str(x) for x in output]) for output in all_model_outputs])}",
level=4)
else:
print_header(f"ID: {subject_id} Score: {model_output}", level=4)
filepaths = get_image_filepath_from_subject_id(subject_id=str(subject_id),
dataset=dataset,
config=config)
if not filepaths:
print_header(
f"Subject ID {subject_id} not found."
f"Note: Reports with datasets that use channel columns in the dataset.csv "
f"are not yet supported.",
level=0)
return
for filepath in filepaths:
success = plot_image_from_filepath(filepath, im_width=im_width)
if not success:
print_header(
"Unable to plot image: image must be 2D with shape [w, h] or [1, w, h].",
level=0)
def print_k_best_and_worst_performing(val_metrics_csv: Path,
test_metrics_csv: Path, k: int,
prediction_target: str) -> None:
"""
Print the top "k" best predictions (i.e. correct classifications where the model was the most certain) and the
top "k" worst predictions (i.e. misclassifications where the model was the most confident).
:param val_metrics_csv: Path to one of the metrics csvs written during inference. This set of metrics will be
used to determine the thresholds for predicting labels on the test set. The best and worst
performing subjects will not be printed out for this csv.
:param test_metrics_csv: Path to one of the metrics csvs written during inference. This is the csv for which
best and worst performing subjects will be printed out.
:param k: Number of subjects of each category to print out.
:param prediction_target: The class label to filter on
"""
results = get_k_best_and_worst_performing(
val_metrics_csv=val_metrics_csv,
test_metrics_csv=test_metrics_csv,
k=k,
prediction_target=prediction_target)
if results is None:
print_header("Empty validation or test set", level=2)
return
print_header(f"Top {k} false positives", level=2)
for index, (subject, model_output) in enumerate(
zip(results.false_positives[LoggingColumns.Patient.value],
results.false_positives[LoggingColumns.ModelOutput.value])):
print_header(f"{index + 1}. ID {subject} Score: {model_output:.5f}",
level=4)
print_header(f"Top {k} false negatives", level=2)
for index, (subject, model_output) in enumerate(
zip(results.false_negatives[LoggingColumns.Patient.value],
results.false_negatives[LoggingColumns.ModelOutput.value])):
print_header(f"{index + 1}. ID {subject} Score: {model_output:.5f}",
level=4)
print_header(f"Top {k} true positives", level=2)
for index, (subject, model_output) in enumerate(
zip(results.true_positives[LoggingColumns.Patient.value],
results.true_positives[LoggingColumns.ModelOutput.value])):
print_header(f"{index + 1}. ID {subject} Score: {model_output:.5f}",
level=4)
print_header(f"Top {k} true negatives", level=2)
for index, (subject, model_output) in enumerate(
zip(results.true_negatives[LoggingColumns.Patient.value],
results.true_negatives[LoggingColumns.ModelOutput.value])):
print_header(f"{index + 1}. ID {subject} Score: {model_output:.5f}",
level=4)
