def test_metrics_dict_with_default_hue() -> None:
hue_name = "foo"
metrics_dict = MetricsDict(hues=[hue_name, MetricsDict.DEFAULT_HUE_KEY])
assert metrics_dict.get_hue_names(include_default=True) == [
hue_name, MetricsDict.DEFAULT_HUE_KEY
]
assert metrics_dict.get_hue_names(include_default=False) == [hue_name]
def test_add_foreground_dice() -> None:
g1 = "Liver"
g2 = "Lung"
ground_truth_ids = [BACKGROUND_CLASS_NAME, g1, g2]
dice = [0.85, 0.75, 0.55]
m = MetricsDict(hues=ground_truth_ids)
for j, ground_truth_id in enumerate(ground_truth_ids):
m.add_metric(MetricType.DICE, dice[j], hue=ground_truth_id)
metrics.add_average_foreground_dice(m)
assert m.get_single_metric(MetricType.DICE) == 0.5 * (dice[1] + dice[2])
def add_average_foreground_dice(metrics: MetricsDict) -> None:
"""
If the given metrics dictionary contains an entry for Dice score, and only one value for the Dice score per class,
then add an average Dice score for all foreground classes to the metrics dictionary (modified in place).
:param metrics: The object that holds metrics. The average Dice score will be written back into this object.
"""
all_dice = []
for structure_name in metrics.get_hue_names(include_default=False):
if structure_name != BACKGROUND_CLASS_NAME:
all_dice.append(metrics.get_single_metric(MetricType.DICE, hue=structure_name))
metrics.add_metric(MetricType.DICE, np.nanmean(all_dice).item())
def test_diagnostics() -> None:
"""
Test if we can store diagnostic values (no restrictions on data types) in the metrics dict.
"""
name = "foo"
value1 = "something"
value2 = (1, 2, 3)
m = MetricsDict()
m.add_diagnostics(name, value1)
m.add_diagnostics(name, value2)
assert m.diagnostics == {name: [value1, value2]}
def test_classification_metrics_avg() -> None:
hue1 = "H1"
hue2 = "H2"
m = MetricsDict(hues=[hue1, hue2], is_classification_metrics=True)
m.add_metric("foo", 1.0)
m.add_metric("foo", 2.0)
# Perfect predictions for hue1, should give AUC == 1.0
m.add_predictions(["S1", "S2"],
np.array([0.0, 1.0]),
np.array([0.0, 1.0]),
hue=hue1)
expected_hue1_auc = 1.0
# Worst possible predictions for hue2, should give AUC == 0.0
m.add_predictions(["S1", "S2"],
np.array([1.0, 0.0]),
np.array([0.0, 1.0]),
hue=hue2)
expected_hue2_auc = 0.0
averaged = m.average(across_hues=False)
g1_averaged = averaged.values(hue=hue1)
assert MetricType.AREA_UNDER_ROC_CURVE.value in g1_averaged
assert g1_averaged[MetricType.AREA_UNDER_ROC_CURVE.value] == [
expected_hue1_auc
]
assert MetricType.AREA_UNDER_PR_CURVE.value in g1_averaged
assert MetricType.SUBJECT_COUNT.value in g1_averaged
assert g1_averaged[MetricType.SUBJECT_COUNT.value] == [2.0]
default_averaged = averaged.values()
assert default_averaged == {"foo": [1.5]}
can_enumerate = list(averaged.enumerate_single_values())
assert len(can_enumerate) >= 8
assert can_enumerate[0] == (hue1, MetricType.AREA_UNDER_ROC_CURVE.value,
1.0)
assert can_enumerate[-1] == (MetricsDict.DEFAULT_HUE_KEY, "foo", 1.5)
g2_averaged = averaged.values(hue=hue2)
assert MetricType.AREA_UNDER_ROC_CURVE.value in g2_averaged
assert g2_averaged[MetricType.AREA_UNDER_ROC_CURVE.value] == [
expected_hue2_auc
]
averaged_across_hues = m.average(across_hues=True)
assert averaged_across_hues.get_hue_names() == [
MetricsDict.DEFAULT_HUE_KEY
]
assert MetricType.AREA_UNDER_ROC_CURVE.value in averaged_across_hues.values(
)
expected_averaged_auc = 0.5 * (expected_hue1_auc + expected_hue2_auc)
assert averaged_across_hues.values()[
MetricType.AREA_UNDER_ROC_CURVE.value] == [expected_averaged_auc]
def get_correct_and_misclassified_examples(val_metrics_csv: Path, test_metrics_csv: Path,
prediction_target: str = "Default") -> Results:
"""
Given the paths to the metrics files for the validation and test sets, get a list of true positives,
false positives, false negatives and true negatives.
The threshold for classification is obtained by looking at the validation file, and applied to the test set to get
label predictions.
The validation and test csvs must have at least the following columns (defined in the LoggingColumns enum):
LoggingColumns.Hue, LoggingColumns.Patient, LoggingColumns.Label, LoggingColumns.ModelOutput.
"""
df_val = read_csv_and_filter_prediction_target(val_metrics_csv, prediction_target)
fpr, tpr, thresholds = roc_curve(df_val[LoggingColumns.Label.value], df_val[LoggingColumns.ModelOutput.value])
optimal_idx = MetricsDict.get_optimal_idx(fpr=fpr, tpr=tpr)
optimal_threshold = thresholds[optimal_idx]
df_test = read_csv_and_filter_prediction_target(test_metrics_csv, prediction_target)
df_test["predicted"] = df_test.apply(lambda x: int(x[LoggingColumns.ModelOutput.value] >= optimal_threshold),
axis=1)
true_positives = df_test[(df_test["predicted"] == 1) & (df_test[LoggingColumns.Label.value] == 1)]
false_positives = df_test[(df_test["predicted"] == 1) & (df_test[LoggingColumns.Label.value] == 0)]
false_negatives = df_test[(df_test["predicted"] == 0) & (df_test[LoggingColumns.Label.value] == 1)]
true_negatives = df_test[(df_test["predicted"] == 0) & (df_test[LoggingColumns.Label.value] == 0)]
return Results(true_positives=true_positives,
true_negatives=true_negatives,
false_positives=false_positives,
false_negatives=false_negatives)
def test_delete_hue() -> None:
h1 = "a"
h2 = "b"
a = MetricsDict(hues=[h1, h2])
a.add_metric("foo", 1.0, hue=h1)
a.add_metric("bar", 2.0, hue=h2)
a.delete_hue(h1)
assert a.get_hue_names(include_default=False) == [h2]
assert list(a.enumerate_single_values()) == [(h2, "bar", 2.0)]
def test_delete_metric() -> None:
"""
Deleting a set of metrics from the dictionary.
"""
m = MetricsDict()
m.add_metric(MetricType.LOSS, 1)
assert m.values()[MetricType.LOSS.value] == [1.0]
m.delete_metric(MetricType.LOSS)
assert MetricType.LOSS.value not in m.values()
def get_metric(predictions_to_set_optimal_threshold: LabelsAndPredictions,
predictions_to_compute_metrics: LabelsAndPredictions,
metric: ReportedScalarMetrics,
optimal_threshold: Optional[float] = None) -> float:
"""
Given LabelsAndPredictions objects for the validation and test sets, return the specified metric.
:param predictions_to_set_optimal_threshold: This set of ground truth labels and model predictions is used to
determine the optimal threshold for classification.
:param predictions_to_compute_metrics: The set of labels and model outputs to calculate metrics for.
:param metric: The name of the metric to calculate.
:param optimal_threshold: If provided, use this threshold instead of calculating an optimal threshold.
"""
if not optimal_threshold:
fpr, tpr, thresholds = roc_curve(
predictions_to_set_optimal_threshold.labels,
predictions_to_set_optimal_threshold.model_outputs)
optimal_idx = MetricsDict.get_optimal_idx(fpr=fpr, tpr=tpr)
optimal_threshold = thresholds[optimal_idx]
assert optimal_threshold # for mypy, we have already calculated optimal threshold if it was set to None
if metric is ReportedScalarMetrics.OptimalThreshold:
return optimal_threshold
only_one_class_present = len(set(
predictions_to_compute_metrics.labels)) < 2
if metric is ReportedScalarMetrics.AUC_ROC:
return math.nan if only_one_class_present else roc_auc_score(
predictions_to_compute_metrics.labels,
predictions_to_compute_metrics.model_outputs)
elif metric is ReportedScalarMetrics.AUC_PR:
if only_one_class_present:
return math.nan
precision, recall, _ = precision_recall_curve(
predictions_to_compute_metrics.labels,
predictions_to_compute_metrics.model_outputs)
return auc(recall, precision)
elif metric is ReportedScalarMetrics.AccuracyAtOptimalThreshold:
return binary_classification_accuracy(
model_output=predictions_to_compute_metrics.model_outputs,
label=predictions_to_compute_metrics.labels,
threshold=optimal_threshold)
elif metric is ReportedScalarMetrics.AccuracyAtThreshold05:
return binary_classification_accuracy(
model_output=predictions_to_compute_metrics.model_outputs,
label=predictions_to_compute_metrics.labels,
threshold=0.5)
elif metric is ReportedScalarMetrics.Specificity:
return recall_score(
predictions_to_compute_metrics.labels,
predictions_to_compute_metrics.model_outputs >= optimal_threshold,
pos_label=0)
elif metric is ReportedScalarMetrics.Sensitivity:
return recall_score(
predictions_to_compute_metrics.labels,
predictions_to_compute_metrics.model_outputs >= optimal_threshold)
else:
raise ValueError("Unknown metric")
def test_metrics_dict_average_additional_metrics() -> None:
"""
Test if computing the ROC entries and metrics at optimal threshold with MetricsDict.average() works
as expected and returns the correct values.
"""
# Prepare a vector of predictions and labels.
predictions = np.array([0.5, 0.6, 0.1, 0.8, 0.2, 0.9])
labels = np.array([0, 1.0, 0, 0, 1, 1], dtype=np.float)
split_length = [3, 2, 1]
# Get MetricsDict
assert sum(split_length) == len(predictions)
summed = np.cumsum(split_length)
# MetricsDict will get that supplied in 3 chunks.
m = MetricsDict()
for i, end in enumerate(summed):
start = 0 if i == 0 else summed[i - 1]
pred = predictions[start:end]
label = labels[start:end]
subject_ids = list(map(str, range(len(pred))))
m.add_predictions(subject_ids, pred, label)
assert m.has_prediction_entries
# Compute average MetricsDict
averaged = m.average()
# Compute additional expected metrics for the averaged MetricsDict
expected_auc = roc_auc_score(labels, predictions)
expected_fpr, expected_tpr, thresholds = roc_curve(labels, predictions)
expected_optimal_idx = np.argmax(expected_tpr - expected_fpr)
expected_optimal_threshold = float(thresholds[expected_optimal_idx])
expected_accuracy = np.mean(
(predictions > expected_optimal_threshold) == labels)
# Check computed values against expected
assert averaged.values()[MetricType.OPTIMAL_THRESHOLD.value][
0] == pytest.approx(expected_optimal_threshold)
assert averaged.values()[MetricType.ACCURACY_AT_OPTIMAL_THRESHOLD.
value][0] == pytest.approx(expected_accuracy)
assert averaged.values()[MetricType.FALSE_POSITIVE_RATE_AT_OPTIMAL_THRESHOLD.value][0] == \
pytest.approx(expected_fpr[expected_optimal_idx])
assert averaged.values()[MetricType.FALSE_NEGATIVE_RATE_AT_OPTIMAL_THRESHOLD.value][0] == \
pytest.approx(1 - expected_tpr[expected_optimal_idx])
assert averaged.values()[
MetricType.AREA_UNDER_ROC_CURVE.value][0] == pytest.approx(
expected_auc, 1e-6)
def test_metrics_dict_get_hues() -> None:
"""
Test to make sure metrics dict is configured properly with/without hues
"""
m = MetricsDict()
assert m.get_hue_names() == [MetricsDict.DEFAULT_HUE_KEY]
assert m.get_hue_names(include_default=False) == []
_hues = ["A", "B", "C"]
m = MetricsDict(hues=_hues)
assert m.get_hue_names() == _hues + [MetricsDict.DEFAULT_HUE_KEY]
assert m.get_hue_names(include_default=False) == _hues
def test_metrics_store_mixed_hues() -> None:
"""
Test to make sure metrics dict is able to handle default and non-default hues
"""
m = MetricsDict(hues=["A", "B"])
m.add_metric("foo", 1)
m.add_metric("foo", 1, hue="B")
m.add_metric("bar", 2, hue="A")
assert list(m.enumerate_single_values()) == \
[('A', 'bar', 2), ('B', 'foo', 1), (MetricsDict.DEFAULT_HUE_KEY, 'foo', 1)]
def test_metrics_dict_roc() -> None:
"""
Test if adding ROC entries to a MetricsDict instance works, and returns the correct AUC.
"""
# Prepare a vector of predictions and labels. We can compute AUC off those to compare.
# MetricsDict will get that supplied in 3 chunks, and should return the same AUC value.
predictions = np.array([0.5, 0.6, 0.1, 0.8, 0.2, 0.9])
labels = np.array([0, 1.0, 0, 0, 1, 1], dtype=np.float)
split_length = [3, 2, 1]
assert sum(split_length) == len(predictions)
summed = np.cumsum(split_length)
m = MetricsDict()
for i, end in enumerate(summed):
start = 0 if i == 0 else summed[i - 1]
pred = predictions[start:end]
label = labels[start:end]
subject_ids = list(map(str, range(len(pred))))
m.add_predictions(subject_ids, pred, label)
assert m.has_prediction_entries
actual_auc = m.get_roc_auc()
expected_auc = roc_auc_score(labels, predictions)
assert actual_auc == pytest.approx(expected_auc, 1e-6)
actual_pr_auc = m.get_pr_auc()
expected_pr_auc = 0.7111111
assert actual_pr_auc == pytest.approx(expected_pr_auc, 1e-6)
def test_metrics_dict_add_integer() -> None:
"""
Adding a scalar metric where the value is an integer by accident should still store the metric.
"""
m = MetricsDict()
m.add_metric("foo", 1)
assert "foo" in m.values()
assert m.values()["foo"] == [1.0]
def test_metrics_dict_to_string_with_hues() -> None:
"""
Test to make sure metrics dict is able to be stringified correctly with hues
"""
m = MetricsDict(hues=["G1"])
m.add_metric("foo", 1.0)
m.add_metric("bar", math.pi, hue="G1")
m.add_metric("baz", 2.0, hue="G1")
info_df = pd.DataFrame(columns=MetricsDict.DATAFRAME_COLUMNS)
info_df = info_df.append(
{
MetricsDict.DATAFRAME_COLUMNS[0]: "G1",
MetricsDict.DATAFRAME_COLUMNS[1]: "bar: 3.1416, baz: 2.0000"
},
ignore_index=True)
info_df = info_df.append(
{
MetricsDict.DATAFRAME_COLUMNS[0]: MetricsDict.DEFAULT_HUE_KEY,
MetricsDict.DATAFRAME_COLUMNS[1]: "foo: 1.0000"
},
ignore_index=True)
assert m.to_string() == tabulate_dataframe(info_df)
assert m.to_string(tabulate=False) == info_df.to_string(index=False)
def test_metrics_dict_flatten(hues: Optional[List[str]]) -> None:
m = MetricsDict(hues=hues)
_hues = hues or [MetricsDict.DEFAULT_HUE_KEY] * 2
m.add_metric("foo", 1.0, hue=_hues[0])
m.add_metric("foo", 2.0, hue=_hues[1])
m.add_metric("bar", 3.0, hue=_hues[0])
m.add_metric("bar", 4.0, hue=_hues[1])
if hues is None:
average = m.average(across_hues=True)
# We should be able to flatten out all the singleton values that the `average` operation returns
all_values = list(average.enumerate_single_values())
assert all_values == [(MetricsDict.DEFAULT_HUE_KEY, "foo", 1.5),
(MetricsDict.DEFAULT_HUE_KEY, "bar", 3.5)]
# When trying to flatten off a dictionary that has two values, this should fail:
with pytest.raises(ValueError) as ex:
list(m.enumerate_single_values())
assert "only hold 1 item" in str(ex)
else:
average = m.average(across_hues=False)
all_values = list(average.enumerate_single_values())
assert all_values == [('A', 'foo', 1.0), ('A', 'bar', 3.0),
('B', 'foo', 2.0), ('B', 'bar', 4.0)]
def get_correct_and_misclassified_examples(val_metrics_csv: Path,
test_metrics_csv: Path) -> Results:
"""
Given the paths to the metrics files for the validation and test sets, get a list of true positives,
false positives, false negatives and true negatives.
The threshold for classification is obtained by looking at the validation file, and applied to the test set to get
label predictions.
"""
df_val = pd.read_csv(val_metrics_csv)
if not df_val[LoggingColumns.Patient.value].is_unique:
raise ValueError(
f"Subject IDs should be unique, but found duplicate entries "
f"in column {LoggingColumns.Patient.value} in the csv file.")
fpr, tpr, thresholds = roc_curve(df_val[LoggingColumns.Label.value],
df_val[LoggingColumns.ModelOutput.value])
optimal_idx = MetricsDict.get_optimal_idx(fpr=fpr, tpr=tpr)
optimal_threshold = thresholds[optimal_idx]
df_test = pd.read_csv(test_metrics_csv)
if not df_test[LoggingColumns.Patient.value].is_unique:
raise ValueError(
f"Subject IDs should be unique, but found duplicate entries "
f"in column {LoggingColumns.Patient.value} in the csv file.")
df_test["predicted"] = df_test.apply(lambda x: int(x[
LoggingColumns.ModelOutput.value] >= optimal_threshold),
axis=1)
true_positives = df_test[(df_test["predicted"] == 1)
& (df_test[LoggingColumns.Label.value] == 1)]
false_positives = df_test[(df_test["predicted"] == 1)
& (df_test[LoggingColumns.Label.value] == 0)]
false_negatives = df_test[(df_test["predicted"] == 0)
& (df_test[LoggingColumns.Label.value] == 1)]
true_negatives = df_test[(df_test["predicted"] == 0)
& (df_test[LoggingColumns.Label.value] == 0)]
return Results(true_positives=true_positives,
true_negatives=true_negatives,
false_positives=false_positives,
false_negatives=false_negatives)
def test_metrics_dict_roc_degenerate() -> None:
"""
Test if adding ROC entries to a MetricsDict instance works, if there is only 1 class present.
"""
# Prepare a vector of predictions and labels. We can compute AUC off those to compare.
# MetricsDict will get that supplied in 3 chunks, and should return the same AUC value.
predictions = np.array([0.5, 0.6, 0.1, 0.8, 0.2, 0.9])
m = MetricsDict()
subject_ids = list(map(str, range(len(predictions))))
m.add_predictions(subject_ids, predictions, np.ones_like(predictions))
assert m.has_prediction_entries
assert m.get_roc_auc() == 1.0
assert m.get_pr_auc() == 1.0
def get_metric(val_metrics_csv: Path, test_metrics_csv: Path,
metric: ReportedMetrics) -> float:
"""
Given a csv file, read the predicted values and ground truth labels and return the specified metric.
"""
results_val = get_results(val_metrics_csv)
fpr, tpr, thresholds = roc_curve(results_val.labels,
results_val.model_outputs)
optimal_idx = MetricsDict.get_optimal_idx(fpr=fpr, tpr=tpr)
optimal_threshold = thresholds[optimal_idx]
if metric is ReportedMetrics.OptimalThreshold:
return optimal_threshold
results_test = get_results(test_metrics_csv)
only_one_class_present = len(set(results_test.labels)) < 2
if metric is ReportedMetrics.AUC_ROC:
return math.nan if only_one_class_present else roc_auc_score(
results_test.labels, results_test.model_outputs)
elif metric is ReportedMetrics.AUC_PR:
if only_one_class_present:
return math.nan
precision, recall, _ = precision_recall_curve(
results_test.labels, results_test.model_outputs)
return auc(recall, precision)
elif metric is ReportedMetrics.Accuracy:
return binary_classification_accuracy(
model_output=results_test.model_outputs,
label=results_test.labels,
threshold=optimal_threshold)
elif metric is ReportedMetrics.FalsePositiveRate:
tnr = recall_score(results_test.labels,
results_test.model_outputs >= optimal_threshold,
pos_label=0)
return 1 - tnr
elif metric is ReportedMetrics.FalseNegativeRate:
return 1 - recall_score(
results_test.labels,
results_test.model_outputs >= optimal_threshold)
else:
raise ValueError("Unknown metric")
def calculate_metrics_per_class(segmentation: np.ndarray,
ground_truth: np.ndarray,
ground_truth_ids: List[str],
voxel_spacing: TupleFloat3,
patient_id: Optional[int] = None) -> MetricsDict:
"""
Calculate the dice for all foreground structures (the background class is completely ignored).
Returns a MetricsDict with metrics for each of the foreground
structures. Metrics are NaN if both ground truth and prediction are all zero for a class.
:param ground_truth_ids: The names of all foreground classes.
:param segmentation: predictions multi-value array with dimensions: [Z x Y x X]
:param ground_truth: ground truth binary array with dimensions: [C x Z x Y x X]
:param voxel_spacing: voxel_spacing in 3D Z x Y x X
:param patient_id: for logging
"""
number_of_classes = ground_truth.shape[0]
if len(ground_truth_ids) != (number_of_classes - 1):
raise ValueError(f"Received {len(ground_truth_ids)} foreground class names, but "
f"the label tensor indicates that there are {number_of_classes - 1} classes.")
binaries = binaries_from_multi_label_array(segmentation, number_of_classes)
all_classes_are_binary = [is_binary_array(ground_truth[label_id]) for label_id in range(ground_truth.shape[0])]
if not np.all(all_classes_are_binary):
raise ValueError("Ground truth values should be 0 or 1")
overlap_measures_filter = sitk.LabelOverlapMeasuresImageFilter()
hausdorff_distance_filter = sitk.HausdorffDistanceImageFilter()
metrics = MetricsDict(hues=ground_truth_ids)
for i, prediction in enumerate(binaries):
if i == 0:
continue
check_size_matches(prediction, ground_truth[i], arg1_name="prediction", arg2_name="ground_truth")
if not is_binary_array(prediction):
raise ValueError("Predictions values should be 0 or 1")
# simpleitk returns a Dice score of 0 if both ground truth and prediction are all zeros.
# We want to be able to fish out those cases, and treat them specially later.
prediction_zero = np.all(prediction == 0)
gt_zero = np.all(ground_truth[i] == 0)
dice = mean_surface_distance = hausdorff_distance = math.nan
if not (prediction_zero and gt_zero):
prediction_image = sitk.GetImageFromArray(prediction.astype(np.uint8))
prediction_image.SetSpacing(sitk.VectorDouble(reverse_tuple_float3(voxel_spacing)))
ground_truth_image = sitk.GetImageFromArray(ground_truth[i].astype(np.uint8))
ground_truth_image.SetSpacing(sitk.VectorDouble(reverse_tuple_float3(voxel_spacing)))
overlap_measures_filter.Execute(prediction_image, ground_truth_image)
dice = overlap_measures_filter.GetDiceCoefficient()
if prediction_zero or gt_zero:
hausdorff_distance = mean_surface_distance = math.inf
else:
try:
hausdorff_distance_filter.Execute(prediction_image, ground_truth_image)
hausdorff_distance = hausdorff_distance_filter.GetHausdorffDistance()
except Exception as e:
logging.warning("Cannot calculate Hausdorff distance for "
f"structure {i} of patient {patient_id}: {e}")
try:
mean_surface_distance = surface_distance(prediction_image, ground_truth_image)
except Exception as e:
logging.warning(f"Cannot calculate mean distance for structure {i} of patient {patient_id}: {e}")
logging.debug(f"Patient {patient_id}, class {i} has Dice score {dice}")
def add_metric(metric_type: MetricType, value: float) -> None:
metrics.add_metric(metric_type, value, skip_nan_when_averaging=True, hue=ground_truth_ids[i - 1])
add_metric(MetricType.DICE, dice)
add_metric(MetricType.HAUSDORFF_mm, hausdorff_distance)
add_metric(MetricType.MEAN_SURFACE_DIST_mm, mean_surface_distance)
return metrics
def test_metrics_dict_average_metrics_averaging() -> None:
"""
Test if averaging metrics avoid NaN as expected.
"""
m = MetricsDict()
metric1 = "foo"
v1 = 1.0
m.add_metric(metric1, v1)
m.add_metric(metric1, np.nan, skip_nan_when_averaging=True)
metric2 = "bar"
v2 = 2.0
m.add_metric(metric2, v2)
m.add_metric(metric2, np.nan, skip_nan_when_averaging=False)
average = m.average()
assert average.values()[metric1] == [v1]
assert np.isnan(average.values()[metric2])
def test_get_single_metric() -> None:
h1 = "a"
m = MetricsDict(hues=[h1])
m1, v1 = ("foo", 1.0)
m2, v2 = (MetricType.LOSS, 2.0)
m.add_metric(m1, v1, hue=h1)
m.add_metric(m2, v2)
assert m.get_single_metric(m1, h1) == v1
assert m.get_single_metric(m2) == v2
with pytest.raises(KeyError) as ex1:
m.get_single_metric(m1, "no such hue")
assert "no such hue" in str(ex1)
with pytest.raises(KeyError) as ex2:
m.get_single_metric("no such metric", h1)
assert "no such metric" in str(ex2)
m.add_metric(m2, v2)
with pytest.raises(ValueError) as ex3:
m.get_single_metric(m2)
assert "Expected a single entry" in str(ex3)
def test_metrics_dict1() -> None:
"""
Test insertion of scalar values into a MetricsDict.
"""
m = MetricsDict()
assert m.get_hue_names() == [MetricsDict.DEFAULT_HUE_KEY]
name = "foo"
v1 = 2.7
v2 = 3.14
m.add_metric(name, v1)
m.add_metric(name, v2)
assert m.values()[name] == [v1, v2]
with pytest.raises(ValueError) as ex:
# noinspection PyTypeChecker
m.add_metric(name, [1.0]) # type: ignore
assert "Expected the metric to be a scalar" in str(ex)
assert m.skip_nan_when_averaging[name] is False
v3 = 3.0
name2 = "bar"
m.add_metric(name2, v3, skip_nan_when_averaging=True)
assert m.skip_nan_when_averaging[name2] is True
# Expected average: Metric "foo" averages over two values v1 and v2. For "bar", we only inserted one value anyhow
average = m.average()
mean_v1_v2 = mean([v1, v2])
assert average.values() == {name: [mean_v1_v2], name2: [v3]}
num_entries = m.num_entries()
assert num_entries == {name: 2, name2: 1}
