def get_rollup_fscore_on_pii( scores: List[TextScore], fbeta: float, recall_threshold: Optional[float] ) -> float: """Calculate f score on PII recognition. A single score, f score, will be calculate to indicate how a system did on predicting PII entities. Recall thresholding is supported, if the system can recognise a certain portion of an entity greater than the threshold, that entity then will be considered identified. Args: scores: a list of text scores providing info including precisions and recalls. fbeta: beta value for f score. recall_threshold: a float between 0 and 1. Any recall value that is greater than or equals to the threshold would be rounded up to 1. Returns: A f score represents performance of a system. """ fscores = [] for text_score in scores: precisions = [p.precision for p in text_score.precisions] recalls = [r.recall for r in text_score.recalls] f = compute_pii_detection_fscore(precisions, recalls, recall_threshold, fbeta) fscores.append(f) if fscores: return round(sum(fscores) / len(fscores), 4) else: # The only possibility to have empty fscores is that argument "scores" # is empty. In this case, we assign f score to 0. return 0.0
def get_rollup_metrics_on_types( grouped_labels: List[Set[str]], scores: List[TextScore], fbeta: float, ) -> Dict[FrozenSet[str], Dict[str, Union[float, str]]]: """Calculate f1, average precision and average recall for every group in the grouped labels. """ score_table = regroup_scores_on_types(grouped_labels, scores) metrics = dict() for key, value in score_table.items(): f1 = round( compute_pii_detection_fscore( value["precisions"], value["recalls"], beta=fbeta ), 4, ) if value["precisions"]: ave_precision = round( sum(value["precisions"]) / len(value["precisions"]), 4 ) else: ave_precision = "undefined" if value["recalls"]: ave_recall = round(sum(value["recalls"]) / len(value["recalls"]), 4) else: ave_recall = "undefined" metrics.update( {key: {"f1": f1, "ave-precision": ave_precision, "ave-recall": ave_recall}} ) return metrics
def test_compute_pii_detection_fscore_with_recall_threshold(): precisions = [0.4, 0.8, 0.9] recalls = [0.2, 0.51, 0.7] actual = compute_pii_detection_fscore(precisions, recalls, recall_threshold=0.5) assert_almost_equal(actual, 0.716279)
def test_compute_pii_detection_fscore_for_empty_precisions_recalls(): actual = compute_pii_detection_fscore([], []) assert actual == 1.0
def test_compute_pii_detection_fscore_for_empty_recalls(): actual = compute_pii_detection_fscore([0.0], []) assert actual == 0.0
def test_compute_pii_detection_fscore_for_empty_precisions(): actual = compute_pii_detection_fscore([], [0.0]) assert actual == 0.0
def test_compute_pii_detection_fscore_for_invalid_threshold(): precisions = recalls = [0.0] with pytest.raises(ValueError) as err: compute_pii_detection_fscore(precisions, recalls, recall_threshold=2.0) assert str(err.value) == ( "Invalid threshold! Recall threshold must between 0 and 1 but got 2.0")
def test_compute_pii_detection_fscore_for_no_recall_threshold(): precisions = [0.4, 0.8] recalls = [0.2, 0.7] actual = compute_pii_detection_fscore(precisions, recalls) assert_almost_equal(actual, 0.5142857)
def test_compute_pii_detection_fscore_for_no_recall_threshold_fscore_is_zero(): precisions = [0.0, 0.0] recalls = [0.0, 0.0] actual = compute_pii_detection_fscore(precisions, recalls) assert actual == 0.0