def test_functions_with_invalid_csv(
test_output_dirs: OutputFolderForTests) -> None:
reports_folder = Path(__file__).parent
test_metrics_file = reports_folder / "test_metrics_classification.csv"
val_metrics_file = reports_folder / "val_metrics_classification.csv"
invalid_metrics_file = Path(
test_output_dirs.root_dir) / "invalid_metrics_classification.csv"
shutil.copyfile(test_metrics_file, invalid_metrics_file)
# Duplicate a subject
with open(invalid_metrics_file, "a") as file:
file.write(f"{MetricsDict.DEFAULT_HUE_KEY},0,5,1.0,1,-1,Test")
with pytest.raises(ValueError) as ex:
get_labels_and_predictions(invalid_metrics_file,
MetricsDict.DEFAULT_HUE_KEY)
assert "Subject IDs should be unique" in str(ex)
with pytest.raises(ValueError) as ex:
get_correct_and_misclassified_examples(invalid_metrics_file,
test_metrics_file,
MetricsDict.DEFAULT_HUE_KEY)
assert "Subject IDs should be unique" in str(ex)
with pytest.raises(ValueError) as ex:
get_correct_and_misclassified_examples(val_metrics_file,
invalid_metrics_file,
MetricsDict.DEFAULT_HUE_KEY)
assert "Subject IDs should be unique" in str(ex)
def test_get_labels_and_predictions() -> None:
reports_folder = Path(__file__).parent
test_metrics_file = reports_folder / "test_metrics_classification.csv"
results = get_labels_and_predictions(test_metrics_file, MetricsDict.DEFAULT_HUE_KEY)
assert all([results.subject_ids[i] == i for i in range(12)])
assert all([results.labels[i] == label for i, label in enumerate([1] * 6 + [0] * 6)])
assert all([results.model_outputs[i] == op for i, op in enumerate([0.0, 0.2, 0.4, 0.6, 0.8, 1.0] * 2)])
def print_metrics_for_thresholded_output_for_all_prediction_targets(
val_metrics_csv: Path, test_metrics_csv: Path,
config: ScalarModelBase) -> None:
"""
Given csvs written during inference for the validation and test sets, print out metrics for every combination of
prediction targets that exist in the dataset (i.e. for every subset of classes that occur in the dataset).
:param val_metrics_csv: Csv written during inference time for the val set. This is used to determine the
optimal threshold for classification.
:param test_metrics_csv: Csv written during inference time for the test set. Metrics are calculated for this csv.
:param config: Model config
"""
unique_prediction_target_combinations = get_unique_prediction_target_combinations(
config)
all_prediction_target_combinations = list(
set([
frozenset([prediction_target])
for prediction_target in config.class_names
])
| unique_prediction_target_combinations)
thresholds_per_prediction_target = []
for label in config.class_names:
val_metrics = get_labels_and_predictions(val_metrics_csv, label)
test_metrics = get_labels_and_predictions(test_metrics_csv, label)
thresholds_per_prediction_target.append(
get_metric(val_labels_and_predictions=val_metrics,
test_labels_and_predictions=test_metrics,
metric=ReportedMetrics.OptimalThreshold))
for labels in all_prediction_target_combinations:
print_header(f"Class {'|'.join(labels) or 'Negative'}", level=3)
val_metrics = get_labels_and_predictions_for_prediction_target_set(
csv=val_metrics_csv,
prediction_target_set_to_match=list(labels),
all_prediction_targets=config.class_names,
thresholds_per_prediction_target=thresholds_per_prediction_target)
test_metrics = get_labels_and_predictions_for_prediction_target_set(
csv=test_metrics_csv,
prediction_target_set_to_match=list(labels),
all_prediction_targets=config.class_names,
thresholds_per_prediction_target=thresholds_per_prediction_target)
print_metrics(val_labels_and_predictions=val_metrics,
test_labels_and_predictions=test_metrics,
is_thresholded=True)
def test_get_metric() -> None:
reports_folder = Path(__file__).parent
test_metrics_file = reports_folder / "test_metrics_classification.csv"
val_metrics_file = reports_folder / "val_metrics_classification.csv"
val_metrics = get_labels_and_predictions(val_metrics_file,
MetricsDict.DEFAULT_HUE_KEY)
test_metrics = get_labels_and_predictions(test_metrics_file,
MetricsDict.DEFAULT_HUE_KEY)
optimal_threshold = get_metric(
predictions_to_compute_metrics=test_metrics,
predictions_to_set_optimal_threshold=val_metrics,
metric=ReportedScalarMetrics.OptimalThreshold)
assert optimal_threshold == 0.6
optimal_threshold = get_metric(
predictions_to_compute_metrics=test_metrics,
predictions_to_set_optimal_threshold=val_metrics,
metric=ReportedScalarMetrics.OptimalThreshold,
optimal_threshold=0.3)
assert optimal_threshold == 0.3
auc_roc = get_metric(predictions_to_compute_metrics=test_metrics,
predictions_to_set_optimal_threshold=val_metrics,
metric=ReportedScalarMetrics.AUC_ROC)
assert auc_roc == 0.5
auc_pr = get_metric(predictions_to_compute_metrics=test_metrics,
predictions_to_set_optimal_threshold=val_metrics,
metric=ReportedScalarMetrics.AUC_PR)
assert math.isclose(auc_pr, 13 / 24, abs_tol=1e-15)
accuracy = get_metric(
predictions_to_compute_metrics=test_metrics,
predictions_to_set_optimal_threshold=val_metrics,
metric=ReportedScalarMetrics.AccuracyAtOptimalThreshold)
assert accuracy == 0.5
accuracy = get_metric(
predictions_to_compute_metrics=test_metrics,
predictions_to_set_optimal_threshold=val_metrics,
metric=ReportedScalarMetrics.AccuracyAtOptimalThreshold,
optimal_threshold=0.1)
assert accuracy == 0.5
accuracy = get_metric(predictions_to_compute_metrics=test_metrics,
predictions_to_set_optimal_threshold=val_metrics,
metric=ReportedScalarMetrics.AccuracyAtThreshold05)
assert accuracy == 0.5
specificity = get_metric(predictions_to_compute_metrics=test_metrics,
predictions_to_set_optimal_threshold=val_metrics,
metric=ReportedScalarMetrics.Specificity)
assert specificity == 0.5
specificity = get_metric(predictions_to_compute_metrics=test_metrics,
predictions_to_set_optimal_threshold=val_metrics,
metric=ReportedScalarMetrics.Specificity,
optimal_threshold=0.1)
assert specificity == 1 / 6
sensitivity = get_metric(predictions_to_compute_metrics=test_metrics,
predictions_to_set_optimal_threshold=val_metrics,
metric=ReportedScalarMetrics.Sensitivity)
assert sensitivity == 0.5
sensitivity = get_metric(predictions_to_compute_metrics=test_metrics,
predictions_to_set_optimal_threshold=val_metrics,
metric=ReportedScalarMetrics.Sensitivity,
optimal_threshold=0.1)
assert math.isclose(sensitivity, 5 / 6, abs_tol=1e-15)
def test_get_labels_and_predictions_with_filters() -> None:
reports_folder = Path(__file__).parent
crossval_metrics_file = reports_folder / "crossval_metrics_classification.csv"
with pytest.raises(ValueError) as ex:
get_labels_and_predictions(crossval_metrics_file,
MetricsDict.DEFAULT_HUE_KEY)
assert "Subject IDs should be unique" in str(ex)
with pytest.raises(ValueError) as ex:
get_labels_and_predictions(crossval_metrics_file,
MetricsDict.DEFAULT_HUE_KEY,
crossval_split_index=0)
assert "Subject IDs should be unique" in str(ex)
with pytest.raises(ValueError) as ex:
get_labels_and_predictions(crossval_metrics_file,
MetricsDict.DEFAULT_HUE_KEY,
data_split=ModelExecutionMode.VAL)
assert "Subject IDs should be unique" in str(ex)
results = get_labels_and_predictions(crossval_metrics_file,
MetricsDict.DEFAULT_HUE_KEY,
crossval_split_index=0,
data_split=ModelExecutionMode.VAL)
assert len(
results.subject_ids
) > 0, "Expected non-empty results for valid crossval_split_index and data_split"
results = get_labels_and_predictions(crossval_metrics_file,
MetricsDict.DEFAULT_HUE_KEY,
crossval_split_index=100,
data_split=ModelExecutionMode.VAL)
assert len(
results.subject_ids
) == 0, "Expected empty results for unavailable crossval_split_index"
results = get_labels_and_predictions(crossval_metrics_file,
MetricsDict.DEFAULT_HUE_KEY,
crossval_split_index=0,
data_split=ModelExecutionMode.TRAIN)
assert len(results.subject_ids
) == 0, "Expected empty results for unavailable data_split"
results = get_labels_and_predictions(crossval_metrics_file,
MetricsDict.DEFAULT_HUE_KEY,
crossval_split_index=0,
data_split=ModelExecutionMode.VAL,
epoch=0)
assert len(results.subject_ids) > 0, "Expected non-empty results for valid crossval_split_index, " \
"data_split, and epoch"
results = get_labels_and_predictions(crossval_metrics_file,
MetricsDict.DEFAULT_HUE_KEY,
crossval_split_index=0,
data_split=ModelExecutionMode.VAL,
epoch=100)
assert len(results.subject_ids
) == 0, "Expected empty results for unavailable epoch"
def test_get_metric() -> None:
reports_folder = Path(__file__).parent
test_metrics_file = reports_folder / "test_metrics_classification.csv"
val_metrics_file = reports_folder / "val_metrics_classification.csv"
val_metrics = get_labels_and_predictions(val_metrics_file, MetricsDict.DEFAULT_HUE_KEY)
test_metrics = get_labels_and_predictions(test_metrics_file, MetricsDict.DEFAULT_HUE_KEY)
optimal_threshold = get_metric(test_labels_and_predictions=test_metrics,
val_labels_and_predictions=val_metrics,
metric=ReportedMetrics.OptimalThreshold)
assert optimal_threshold == 0.6
optimal_threshold = get_metric(test_labels_and_predictions=test_metrics,
val_labels_and_predictions=val_metrics,
metric=ReportedMetrics.OptimalThreshold,
optimal_threshold=0.3)
assert optimal_threshold == 0.3
auc_roc = get_metric(test_labels_and_predictions=test_metrics,
val_labels_and_predictions=val_metrics,
metric=ReportedMetrics.AUC_ROC)
assert auc_roc == 0.5
auc_pr = get_metric(test_labels_and_predictions=test_metrics,
val_labels_and_predictions=val_metrics,
metric=ReportedMetrics.AUC_PR)
assert math.isclose(auc_pr, 13 / 24, abs_tol=1e-15)
accuracy = get_metric(test_labels_and_predictions=test_metrics,
val_labels_and_predictions=val_metrics,
metric=ReportedMetrics.Accuracy)
assert accuracy == 0.5
accuracy = get_metric(test_labels_and_predictions=test_metrics,
val_labels_and_predictions=val_metrics,
metric=ReportedMetrics.Accuracy,
optimal_threshold=0.1)
assert accuracy == 0.5
fpr = get_metric(test_labels_and_predictions=test_metrics,
val_labels_and_predictions=val_metrics,
metric=ReportedMetrics.FalsePositiveRate)
assert fpr == 0.5
fpr = get_metric(test_labels_and_predictions=test_metrics,
val_labels_and_predictions=val_metrics,
metric=ReportedMetrics.FalsePositiveRate,
optimal_threshold=0.1)
assert fpr == 5 / 6
fnr = get_metric(test_labels_and_predictions=test_metrics,
val_labels_and_predictions=val_metrics,
metric=ReportedMetrics.FalseNegativeRate)
assert fnr == 0.5
fnr = get_metric(test_labels_and_predictions=test_metrics,
val_labels_and_predictions=val_metrics,
metric=ReportedMetrics.FalseNegativeRate,
optimal_threshold=0.1)
assert math.isclose(fnr, 1 / 6, abs_tol=1e-15)