def calculate_precisions_and_recalls( data: Data, grouped_targeted_labels: List[Set[str]], nontargeted_labels: Optional[Set[str]] = None, ) -> Dict[str, List[TextScore]]: label_mapping = build_label_mapping(grouped_targeted_labels, nontargeted_labels) scores = [] for item in data.items: if item.pred_labels: pred_labels = item.pred_labels else: # pred_labels could be None pred_labels = [] ent_precisions = compute_entity_precisions_for_prediction( len(item.text), item.true_labels, pred_labels, label_mapping ) ent_recalls = compute_entity_recalls_for_ground_truth( len(item.text), item.true_labels, pred_labels, label_mapping ) ticket_score = TextScore( text=item.text, precisions=ent_precisions, recalls=ent_recalls ) scores.append(ticket_score) return {"scores": scores}
def test_compute_precisions_recalls_for_exact_match(): true_entities = pred_entities = [ Entity(entity_type="LOC", start=3, end=7), Entity(entity_type="PER", start=10, end=15), Entity(entity_type="LOC", start=23, end=32), Entity(entity_type="PER", start=37, end=48), ] # type matters label_to_int = {"LOC": 1, "PER": 2} precisions = compute_entity_precisions_for_prediction( 50, true_entities, pred_entities, label_to_int) recalls = compute_entity_recalls_for_ground_truth(50, true_entities, pred_entities, label_to_int) assert precisions == [ EntityPrecision(Entity(entity_type="LOC", start=3, end=7), 1.0), EntityPrecision(Entity(entity_type="PER", start=10, end=15), 1.0), EntityPrecision(Entity(entity_type="LOC", start=23, end=32), 1.0), EntityPrecision(Entity(entity_type="PER", start=37, end=48), 1.0), ] assert recalls == [ EntityRecall(Entity(entity_type="LOC", start=3, end=7), 1.0), EntityRecall(Entity(entity_type="PER", start=10, end=15), 1.0), EntityRecall(Entity(entity_type="LOC", start=23, end=32), 1.0), EntityRecall(Entity(entity_type="PER", start=37, end=48), 1.0), ]
def test_compute_precisions_recalls_for_pred_subset_of_true(): # Every predicted entity is a subset of one true entity true_entities = pred_entities = [ Entity(entity_type="LOC", start=3, end=7), Entity(entity_type="PER", start=10, end=15), Entity(entity_type="LOC", start=23, end=32), Entity(entity_type="PER", start=37, end=48), ] pred_entities = [ Entity(entity_type="LOC", start=4, end=7), Entity(entity_type="PER", start=13, end=15), Entity(entity_type="LOC", start=25, end=30), Entity(entity_type="PER", start=40, end=46), ] # type matters label_to_int = {"LOC": 1, "PER": 2} precisions = compute_entity_precisions_for_prediction( 50, true_entities, pred_entities, label_to_int) recalls = compute_entity_recalls_for_ground_truth(50, true_entities, pred_entities, label_to_int) assert precisions == [ EntityPrecision(Entity(entity_type="LOC", start=4, end=7), 1.0), EntityPrecision(Entity(entity_type="PER", start=13, end=15), 1.0), EntityPrecision(Entity(entity_type="LOC", start=25, end=30), 1.0), EntityPrecision(Entity(entity_type="PER", start=40, end=46), 1.0), ] assert recalls == [ EntityRecall(Entity(entity_type="LOC", start=3, end=7), 0.75), EntityRecall(Entity(entity_type="PER", start=10, end=15), 0.4), EntityRecall(Entity(entity_type="LOC", start=23, end=32), 5 / 9), EntityRecall(Entity(entity_type="PER", start=37, end=48), 6 / 11), ]
def test_compute_entity_precisions_for_prediction_no_pred_entities(): actual = compute_entity_precisions_for_prediction( 50, [Entity("PER", 5, 10), Entity("LOC", 15, 25)], [], { "PER": 1, "LOC": 2 }) assert actual == []
def test_compute_precisions_recalls_for_type_doesnt_matter(): # Overlaps but wrong type true_entities = [ Entity(entity_type="LOC", start=3, end=7), Entity(entity_type="PER", start=10, end=15), ] pred_entities = [ Entity(entity_type="PER", start=3, end=7), Entity(entity_type="LOC", start=10, end=15), ] # type DOES NOT matter label_to_int = {"LOC": 1, "PER": 1} precisions = compute_entity_precisions_for_prediction( 50, true_entities, pred_entities, label_to_int) recalls = compute_entity_recalls_for_ground_truth(50, true_entities, pred_entities, label_to_int) assert precisions == [ EntityPrecision(Entity(entity_type="PER", start=3, end=7), 1.0), EntityPrecision(Entity(entity_type="LOC", start=10, end=15), 1.0), ] assert recalls == [ EntityRecall(Entity(entity_type="LOC", start=3, end=7), 1.0), EntityRecall(Entity(entity_type="PER", start=10, end=15), 1.0), ]
def test_compute_precisions_recalls_for_many_preds_to_one_true(): # Every true entity overlaps more than one predicted entities # but its type may not match with all overlapping predicted entities true_entities = [ Entity(entity_type="LOC", start=3, end=20), Entity(entity_type="PER", start=28, end=43), ] pred_entities = pred_entities = [ Entity(entity_type="LOC", start=3, end=7), Entity(entity_type="PER", start=10, end=15), Entity(entity_type="LOC", start=23, end=32), Entity(entity_type="PER", start=37, end=48), ] # type matters label_to_int = {"LOC": 1, "PER": 2} precisions = compute_entity_precisions_for_prediction( 50, true_entities, pred_entities, label_to_int) recalls = compute_entity_recalls_for_ground_truth(50, true_entities, pred_entities, label_to_int) assert precisions == [ EntityPrecision(Entity(entity_type="LOC", start=3, end=7), 1.0), EntityPrecision(Entity(entity_type="PER", start=10, end=15), 0.0), EntityPrecision(Entity(entity_type="LOC", start=23, end=32), 0.0), EntityPrecision(Entity(entity_type="PER", start=37, end=48), 6 / 11), ] assert recalls == [ EntityRecall(Entity(entity_type="LOC", start=3, end=20), 4 / 17), EntityRecall(Entity(entity_type="PER", start=28, end=43), 0.4), ]
def test_compute_precisions_recalls_for_no_overlap(): # No predicted entity overlaps with any true entity true_entities = pred_entities = [ Entity(entity_type="LOC", start=3, end=7), Entity(entity_type="PER", start=10, end=15), Entity(entity_type="LOC", start=23, end=32), Entity(entity_type="PER", start=37, end=48), ] pred_entities = [ Entity(entity_type="LOC", start=15, end=20), Entity(entity_type="PER", start=33, end=35), ] # type matters label_to_int = {"LOC": 1, "PER": 2} precisions = compute_entity_precisions_for_prediction( 50, true_entities, pred_entities, label_to_int) recalls = compute_entity_recalls_for_ground_truth(50, true_entities, pred_entities, label_to_int) assert precisions == [ EntityPrecision(Entity(entity_type="LOC", start=15, end=20), 0.0), EntityPrecision(Entity(entity_type="PER", start=33, end=35), 0.0), ] assert recalls == [ EntityRecall(Entity(entity_type="LOC", start=3, end=7), 0.0), EntityRecall(Entity(entity_type="PER", start=10, end=15), 0.0), EntityRecall(Entity(entity_type="LOC", start=23, end=32), 0.0), EntityRecall(Entity(entity_type="PER", start=37, end=48), 0.0), ]
def test_compute_precisions_recalls_for_pred_overlap_true(): # Every predicted entity overlaps with one true entity true_entities = pred_entities = [ Entity(entity_type="LOC", start=3, end=7), Entity(entity_type="PER", start=10, end=15), Entity(entity_type="LOC", start=23, end=32), Entity(entity_type="PER", start=37, end=48), ] pred_entities = [ Entity(entity_type="LOC", start=1, end=4), Entity(entity_type="PER", start=13, end=18), Entity(entity_type="LOC", start=28, end=35), Entity(entity_type="PER", start=45, end=49), ] # type matters label_to_int = {"LOC": 1, "PER": 2} precisions = compute_entity_precisions_for_prediction( 50, true_entities, pred_entities, label_to_int) recalls = compute_entity_recalls_for_ground_truth(50, true_entities, pred_entities, label_to_int) assert precisions == [ EntityPrecision(Entity(entity_type="LOC", start=1, end=4), 1 / 3), EntityPrecision(Entity(entity_type="PER", start=13, end=18), 0.4), EntityPrecision(Entity(entity_type="LOC", start=28, end=35), 4 / 7), EntityPrecision(Entity(entity_type="PER", start=45, end=49), 0.75), ] assert recalls == [ EntityRecall(Entity(entity_type="LOC", start=3, end=7), 0.25), EntityRecall(Entity(entity_type="PER", start=10, end=15), 0.4), EntityRecall(Entity(entity_type="LOC", start=23, end=32), 4 / 9), EntityRecall(Entity(entity_type="PER", start=37, end=48), 3 / 11), ]
def test_compute_precisions_recalls_for_no_true(): true_entities: List = [] pred_entities = [ Entity(entity_type="LOC", start=3, end=7), Entity(entity_type="PER", start=10, end=15), ] # type matters label_to_int = {"LOC": 1, "PER": 2} precisions = compute_entity_precisions_for_prediction( 50, true_entities, pred_entities, label_to_int) recalls = compute_entity_recalls_for_ground_truth(50, true_entities, pred_entities, label_to_int) assert precisions == [ EntityPrecision(Entity(entity_type="LOC", start=3, end=7), 0.0), EntityPrecision(Entity(entity_type="PER", start=10, end=15), 0.0), ] assert recalls == []
def test_compute_entity_precisions_for_prediction_no_true_no_pred_entities(): actual = compute_entity_precisions_for_prediction(50, [], [], { "PER": 1, "LOC": 2 }) assert actual == []