def print_metrics(self, report_pep=False) -> None:
print(f"Accuracy: {self.accuracy * 100:.2f}\n")
print("Macro P/R/F1 Scores:")
print("\nSoft Metrics:")
if self.per_label_soft_scores:
soft_scores = [
{
"label": label,
"avg_pr": f"{metrics.average_precision:.3f}",
"roc_auc": f"{(metrics.roc_auc or 0.0):.3f}",
}
for label, metrics in sorted(self.per_label_soft_scores.items())
]
columns = {
"label": "Label",
"avg_pr": "Average precision",
"roc_auc": "ROC AUC",
}
print(ascii_table(soft_scores, columns))
all_thresholds = set(
itertools.chain.from_iterable(
metrics.recall_at_precision
for metrics in self.per_label_soft_scores.values()
)
)
print("\nRecall at Precision")
print(
ascii_table(
(
dict(
{"label": label},
**{
str(p): f"{r:.3f}"
for p, r in metrics.recall_at_precision.items()
},
)
for label, metrics in sorted(self.per_label_soft_scores.items())
),
dict(
{"label": "Label"},
**{str(t): f"R@P {t}" for t in all_thresholds},
),
alignments={"label": "<"},
)
)
if self.mcc:
print(f"\nMatthews correlation coefficient: {self.mcc :.3f}")
if self.roc_auc:
print(f"\nROC AUC: {self.roc_auc:.3f}")
if report_pep:
self.print_pep()
def print_metrics(self, indentation="") -> None:
print(
ascii_table(
[{
"label": label,
"precision": f"{metrics.precision:.2f}",
"recall": f"{metrics.recall:.2f}",
"f1": f"{metrics.f1:.2f}",
"support":
metrics.true_positives + metrics.false_negatives,
} for label, metrics in sorted(self.per_label_scores.items())],
human_column_names={
"label": "Label",
"precision": "Precision",
"recall": "Recall",
"f1": "F1",
"support": "Support",
},
footer={
"label": "Overall macro scores",
"precision": f"{self.macro_scores.precision:.2f}",
"recall": f"{self.macro_scores.recall:.2f}",
"f1": f"{self.macro_scores.f1:.2f}",
},
alignments={"label": "<"},
indentation=indentation,
))
def test_with_headers_and_footer(self):
two_rows = [
{
"stage": "forward",
"total": "0.011"
},
{
"stage": "comput_loss",
"total": "0.000"
},
]
headers = {"stage": "Stage", "total": "Total Time"}
footers = {
"stage": "This very long footer is necessary.",
"total": "0.055"
}
in_table_form = ascii_table(two_rows, headers, footers)
output = ("+-------------------------------------+------------+\n"
"| Stage | Total Time |\n"
"+-------------------------------------+------------+\n"
"| forward | 0.011 |\n"
"| comput_loss | 0.000 |\n"
"+-------------------------------------+------------+\n"
"| This very long footer is necessary. | 0.055 |\n"
"+-------------------------------------+------------+")
self.assertEqual(in_table_form, output)
def test_simple(self):
two_rows = [
{
"stage": "forward",
"total": "0.011"
},
{
"stage": "comput_loss",
"total": "0.000"
},
]
in_table_form = ascii_table(two_rows)
output = ("+-------------+-------+\n"
"| forward | 0.011 |\n"
"| comput_loss | 0.000 |\n"
"+-------------+-------+")
self.assertEqual(in_table_form, output)
def print_metrics(self) -> None:
print(f"Accuracy: {self.accuracy * 100:.2f}\n")
print("Macro P/R/F1 Scores:")
self.macro_prf1_metrics.print_metrics(indentation="\t")
print("\nSoft Metrics:")
if self.per_label_soft_scores:
soft_scores = {
label: f"{metrics.average_precision * 100:.2f}"
for label, metrics in self.per_label_soft_scores.items()
}
print(
ascii_table_from_dict(soft_scores,
"Label",
"Average precision",
indentation="\t"))
all_thresholds = set(
itertools.chain.from_iterable(
metrics.recall_at_precision
for metrics in self.per_label_soft_scores.values()))
print("\n\t Precision at Recall")
print(
ascii_table(
(dict(
{"label": label},
**{
str(p): f"{r:.2f}"
for p, r in metrics.recall_at_precision.items()
},
)
for label, metrics in self.per_label_soft_scores.items()),
dict(
{"label": "Label"},
**{str(t): f"P@R {t}"
for t in all_thresholds},
),
indentation="\t",
))
if self.mcc:
print(f"\nMatthews correlation coefficient: {self.mcc :.2f}")
if self.roc_auc:
print(f"\nROC AUC: {self.roc_auc:.3f}")
def test_with_headers(self):
two_rows = [
{
"stage": "forward",
"total": "0.011"
},
{
"stage": "comput_loss",
"total": "0.000"
},
]
headers = {"stage": "Stage", "total": "Total Time"}
in_table_form = ascii_table(two_rows, headers)
output = ("+-------------+------------+\n"
"| Stage | Total Time |\n"
"+-------------+------------+\n"
"| forward | 0.011 |\n"
"| comput_loss | 0.000 |\n"
"+-------------+------------+")
self.assertEqual(in_table_form, output)
def print_metrics(self, report_pep=False) -> None:
print(f"Accuracy: {self.accuracy * 100:.2f}")
print("\nSoft Metrics:")
if self.per_label_soft_scores:
soft_scores = []
for label, metrics in sorted(self.per_label_soft_scores.items()):
total_num_examples = 0
true_positive = 0
false_positive = 0
false_negative = 0
if label in self.macro_prf1_metrics.per_label_scores:
per_label_score = self.macro_prf1_metrics.per_label_scores[label]
true_positive = per_label_score.true_positives
false_positive = per_label_score.false_positives
false_negative = per_label_score.false_negatives
total_num_examples = (
per_label_score.true_positives + per_label_score.false_negatives
)
soft_scores.append(
{
"label": label,
"avg_pr": f"{metrics.average_precision:.3f}",
"roc_auc": f"{(metrics.roc_auc or 0.0):.3f}",
"true_positive": f"{true_positive}",
"false_positive": f"{false_positive}",
"false_negative": f"{false_negative}",
"support": f"{total_num_examples}",
}
)
columns = {
"label": "Label",
"avg_pr": "Average precision",
"roc_auc": "ROC AUC",
"true_positive": "True positive",
"false_positive": "False positive",
"false_negative": "False negative",
"support": "Support",
}
print(ascii_table(soft_scores, columns))
print("\nRecall at Precision")
r_at_p_thresholds = set(
itertools.chain.from_iterable(
metrics.recall_at_precision
for metrics in self.per_label_soft_scores.values()
)
)
print(
ascii_table(
(
dict(
{"label": label},
**{
str(p): f"{r:.3f}"
for p, r in metrics.recall_at_precision.items()
},
)
for label, metrics in sorted(self.per_label_soft_scores.items())
),
dict(
{"label": "Label"},
**{str(t): f"R@P {t}" for t in r_at_p_thresholds},
),
alignments={"label": "<"},
)
)
print("\nPrecision at Recall")
p_at_r_thresholds = set(
itertools.chain.from_iterable(
metrics.precision_at_recall
for metrics in self.per_label_soft_scores.values()
)
)
print(
ascii_table(
(
dict(
{"label": label},
**{
str(p): f"{r:.3f}"
for p, r in metrics.precision_at_recall.items()
},
)
for label, metrics in sorted(self.per_label_soft_scores.items())
),
dict(
{"label": "Label"},
**{str(t): f"P@R {t}" for t in p_at_r_thresholds},
),
alignments={"label": "<"},
)
)
if self.mcc:
print(f"\nMatthews correlation coefficient: {self.mcc :.3f}")
if self.roc_auc:
print(f"\nROC AUC: {self.roc_auc:.3f}")
if report_pep:
self.print_pep()
