def test_DocumentLevelRelationEvaluator_parts_irrelevant(self): evaluator = DocumentLevelRelationEvaluator(rel_type=STUB_R_ID_1) dataset = Dataset() doc_1 = Document() part_1 = Part('_irrelevant_ PART *1*') dataset.documents['doc_1'] = doc_1 doc_1.parts['part_1'] = part_1 part_2 = Part('_irrelevant_ PART *2*') dataset.documents['doc_1'] = doc_1 doc_1.parts['part_2'] = part_2 part_1.relations = [ Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "TOOL"), Entity(STUB_E_ID_2, 0, "maynard")), ] # - part_2.predicted_relations = [ Relation(STUB_R_ID_1, Entity(STUB_E_ID_2, 0, "maynard"), Entity(STUB_E_ID_1, 0, "TOOL")), ] self._apply_pipeline(dataset) # --- evals = evaluator.evaluate(dataset) evaluation = evals(STUB_R_ID_1) self.assertEqual(evaluation.tp, 1) self.assertEqual(evaluation.fn, 0) self.assertEqual(evaluation.fp, 0) computation = evals(STUB_R_ID_1).compute(strictness="exact") self.assertEqual(computation.f_measure, 1.0)
def test_DocumentLevelRelationEvaluator_default_entities_case_irrelevant( self): evaluator = DocumentLevelRelationEvaluator(rel_type=STUB_R_ID_1) dataset = Dataset() doc_1 = Document() part_1 = Part('_irrelevant_') dataset.documents['doc_1'] = doc_1 doc_1.parts['part_1'] = part_1 part_1.relations = [ Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "TOOL"), Entity(STUB_E_ID_2, 0, "maynard")), ] # - part_1.predicted_relations = [ # empty ] self._apply_pipeline(dataset) # - evals = evaluator.evaluate(dataset) evaluation = evals(STUB_R_ID_1) self.assertEqual(evaluation.tp, 0) computation = evals(STUB_R_ID_1).compute(strictness="exact") self.assertEqual(computation.f_measure, 0.0) # --- part_1.predicted_relations = [ Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "TOOL"), Entity(STUB_E_ID_2, 0, "maynard")), ] evals = evaluator.evaluate(dataset) evaluation = evals(STUB_R_ID_1) self.assertEqual(evaluation.tp, 1) computation = evals(STUB_R_ID_1).compute(strictness="exact") self.assertEqual(computation.f_measure, 1.0) # - part_1.predicted_relations = [ Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "tool"), Entity(STUB_E_ID_2, 0, "MAYNARD")), ] evals = evaluator.evaluate(dataset) evaluation = evals(STUB_R_ID_1) self.assertEqual(evaluation.tp, 1) computation = evals(STUB_R_ID_1).compute(strictness="exact") self.assertEqual(computation.f_measure, 1.0)
def test_DocumentLevelRelationEvaluator_arbitrary_relation_accept_fun_order_does_not_matter(self): entity_map_fun = (lambda e: "SAME") def relation_accept_fun(gold, pred): print('gold:', gold, ' <---> ', 'pred:', pred) return gold == pred r1 = Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "yin"), Entity(STUB_E_ID_2, 0, "yan")) r2 = Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "yan"), Entity(STUB_E_ID_2, 0, "yin")) self.assertTrue(relation_accept_fun(r1.map(entity_map_fun), r1.map(entity_map_fun))) self.assertTrue(relation_accept_fun(r1.map(entity_map_fun), r2.map(entity_map_fun))) self.assertTrue(relation_accept_fun(r2.map(entity_map_fun), r1.map(entity_map_fun))) evaluator = DocumentLevelRelationEvaluator(STUB_R_ID_1, entity_map_fun, relation_accept_fun) (dataset, part) = self._create_basic_dataset() # - part.relations = [r1] part.predicted_relations = [r1] evals = evaluator.evaluate(dataset) evaluation = evals(STUB_R_ID_1) print(evaluation) self.assertEqual(evaluation.tp, 1) self.assertEqual(evaluation.fn, 0) self.assertEqual(evaluation.fp, 0) computation = evals(STUB_R_ID_1).compute(strictness="exact") self.assertEqual(computation.f_measure, 1.0)
def __read_annjson(self, reader, filename, dataset): try: doc_id = os.path.basename(filename).replace('.ann.json', '').replace('.json', '') if not self.whole_basename_as_docid and '-' in doc_id: doc_id = doc_id.split('-')[-1] ann_json = json.load(reader) try: document = dataset.documents[doc_id] except Exception as err: print_warning("The annjson with docid={} was not in the whole plain dataset.".format(doc_id)) return doc_id if not (ann_json['anncomplete'] or self.is_predicted) and self.delete_incomplete_docs: del dataset.documents[doc_id] else: for e in ann_json['entities']: if self.read_only_class_id is None or e['classId'] in self.read_only_class_id: part = document.parts[e['part']] try: normalizations = {key: obj['source']['id'] for key, obj in e['normalizations'].items()} except KeyError as err: print_warning("The normalization is badly formatted: (docid={}) {}".format(doc_id, str(e['normalizations']))) normalizations = None entity = Entity( e['classId'], e['offsets'][0]['start'], e['offsets'][0]['text'], e['confidence']['prob'], norms=normalizations) if self.is_predicted: part.predicted_annotations.append(entity) else: part.annotations.append(entity) if self.read_relations: for relation in ann_json['relations']: # Note: no distinction with predicted_relations yet part = document.parts[relation['entities'][0].split('|')[0]] e1_start = int(relation['entities'][0].split('|')[1].split(',')[0]) e2_start = int(relation['entities'][1].split('|')[1].split(',')[0]) rel_id = relation['classId'] e1 = part.get_entity(e1_start, use_pred=False, raise_exception_on_incosistencies=self.raise_exception_on_incosistencies) e2 = part.get_entity(e2_start, use_pred=False, raise_exception_on_incosistencies=self.raise_exception_on_incosistencies) if (not self.raise_exception_on_incosistencies and (e1 is None or e2 is None)): continue rel = Relation(rel_id, e1, e2) part.relations.append(rel) # delete parts that are not annotatable annotatable_parts = set(ann_json['annotatable']['parts']) part_ids_to_del = [] for part_id, part in document.parts.items(): if part_id not in annotatable_parts: part_ids_to_del.append(part_id) for part_id in part_ids_to_del: del document.parts[part_id] return doc_id except Exception as err: if self.raise_exception_on_incosistencies: raise err else: pass
def test_DocumentLevelRelationEvaluator_arbitrary_relation_accept_fun_dont_count_multiple_same_hits(self): entity_map_fun = (lambda e: e.text) def relation_accept_fun(gold, pred): print('gold:', gold, ' <---> ', 'pred:', pred,) gold = int(gold[-1]) pred = int(pred[-1]) if gold <= pred and ((pred - gold) < 3): # e.g., 1 <= 1, 2, 3 return True else: return False return gold == pred r1 = Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "xxx"), Entity(STUB_E_ID_2, 0, "1")) r5 = Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "xxx"), Entity(STUB_E_ID_2, 0, "9")) # Missing == fn r6 = Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "xxx"), Entity(STUB_E_ID_2, 0, "5")) r8 = Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "xxx"), Entity(STUB_E_ID_2, 0, "2")) # (maps to 1) Own repetition in gold, so 1 should be counted twice r2 = Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "xxx"), Entity(STUB_E_ID_2, 0, "1")) # Accept 1 --> do count == tp r3 = Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "xxx"), Entity(STUB_E_ID_2, 0, "2")) # repeated Accept 1,2 --> do count because of own repetition in gold == tp r4 = Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "xxx"), Entity(STUB_E_ID_2, 0, "3")) # repeated Accept 1,2 --> do not count because it's over repetition r7 = Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "xxx"), Entity(STUB_E_ID_2, 0, "6")) # Accept 5 --> do count == tp r9 = Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "xxx"), Entity(STUB_E_ID_2, 0, "5")) # Accept 5 --> do not count because it's over repetition self.assertEqual(True, relation_accept_fun(r1.map(entity_map_fun), r2.map(entity_map_fun))) self.assertEqual(True, relation_accept_fun(r1.map(entity_map_fun), r3.map(entity_map_fun))) self.assertEqual(True, relation_accept_fun(r1.map(entity_map_fun), r4.map(entity_map_fun))) self.assertEqual(False, relation_accept_fun(r1.map(entity_map_fun), r7.map(entity_map_fun))) self.assertEqual(False, relation_accept_fun(r5.map(entity_map_fun), r2.map(entity_map_fun))) self.assertEqual(False, relation_accept_fun(r5.map(entity_map_fun), r3.map(entity_map_fun))) self.assertEqual(False, relation_accept_fun(r5.map(entity_map_fun), r4.map(entity_map_fun))) self.assertEqual(False, relation_accept_fun(r5.map(entity_map_fun), r7.map(entity_map_fun))) self.assertEqual(True, relation_accept_fun(r6.map(entity_map_fun), r7.map(entity_map_fun))) self.assertEqual(False, relation_accept_fun(r8.map(entity_map_fun), r2.map(entity_map_fun))) self.assertEqual(True, relation_accept_fun(r8.map(entity_map_fun), r3.map(entity_map_fun))) self.assertEqual(True, relation_accept_fun(r8.map(entity_map_fun), r4.map(entity_map_fun))) self.assertEqual(False, relation_accept_fun(r8.map(entity_map_fun), r7.map(entity_map_fun))) evaluator = DocumentLevelRelationEvaluator(STUB_R_ID_1, entity_map_fun, relation_accept_fun) (dataset, part) = self._create_basic_dataset() # - part.relations = [r1, r5, r6, r8] part.predicted_relations = [r2, r3, r4, r7, r9] # Only one shold be accepted evals = evaluator.evaluate(dataset) evaluation = evals(STUB_R_ID_1) print(evaluation) self.assertEqual(evaluation.tp, 3, evaluation) self.assertEqual(evaluation.fn, 1) self.assertEqual(evaluation.fp, 0) computation = evals(STUB_R_ID_1).compute(strictness="exact") self.assertEqual(computation.f_measure, 0.8571428571428571)
def test_DocumentLevelRelationEvaluator_arbitrary_relation_accept_fun_ignore_some_predictions(self): entity_map_fun = (lambda e: e.text) def relation_accept_fun(gold, pred): gold_pred_char_num = int(gold[-1]) pred_last_char_num = int(pred[-1]) print('gold:', gold, ' <---> ', 'pred:', pred,) if gold == pred: # 1 == 1 return True elif gold < pred: # 1 < 2 return None else: return False # 1 !<= 0 return gold == pred r1 = Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "xxx"), Entity(STUB_E_ID_2, 0, "1")) r2 = Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "xxx"), Entity(STUB_E_ID_2, 0, "1")) # Accept r3 = Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "xxx"), Entity(STUB_E_ID_2, 0, "2")) # Ignore r4 = Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "xxx"), Entity(STUB_E_ID_2, 0, "0")) # Reject self.assertEqual(True, relation_accept_fun(r1.map(entity_map_fun), r2.map(entity_map_fun))) self.assertEqual(None, relation_accept_fun(r1.map(entity_map_fun), r3.map(entity_map_fun))) self.assertEqual(False, relation_accept_fun(r1.map(entity_map_fun), r4.map(entity_map_fun))) evaluator = DocumentLevelRelationEvaluator(STUB_R_ID_1, entity_map_fun, relation_accept_fun) (dataset, part) = self._create_basic_dataset() # - part.relations = [r1] part.predicted_relations = [r2, r4] + [r3, r3, r3, r3, r3] # All the r3's should be ignored evals = evaluator.evaluate(dataset) evaluation = evals(STUB_R_ID_1) print(evaluation) self.assertEqual(evaluation.tp, 1) self.assertEqual(evaluation.fn, 0) self.assertEqual(evaluation.fp, 1) computation = evals(STUB_R_ID_1).compute(strictness="exact") self.assertEqual(computation.f_measure, 0.6666666666666666)
def test_DocumentLevelRelationEvaluator_arbitrary_relation_accept_fun_order_matters(self): entity_map_fun = (lambda e: e.text) def relation_accept_fun(gold, pred): print('gold:', gold, ' <---> ', 'pred:', pred) return gold < pred r1 = Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "1"), Entity(STUB_E_ID_2, 0, "2")) r2 = Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "2"), Entity(STUB_E_ID_2, 0, "1")) # r1 not equiv r1 because this IS NOT equals (r1 not < r1) self.assertFalse(relation_accept_fun(r1.map(entity_map_fun), r1.map(entity_map_fun))) # r1 < r2 self.assertTrue(relation_accept_fun(r1.map(entity_map_fun), r2.map(entity_map_fun))) # r2 not < r1 self.assertFalse(relation_accept_fun(r2.map(entity_map_fun), r1.map(entity_map_fun))) evaluator = DocumentLevelRelationEvaluator(STUB_R_ID_1, entity_map_fun, relation_accept_fun) (dataset, part) = self._create_basic_dataset() # - part.relations = [r1] part.predicted_relations = [r1] evals = evaluator.evaluate(dataset) evaluation = evals(STUB_R_ID_1) print(evaluation) self.assertEqual(evaluation.tp, 0) self.assertEqual(evaluation.fn, 1) self.assertEqual(evaluation.fp, 1) computation = evals(STUB_R_ID_1).compute(strictness="exact") self.assertEqual(computation.f_measure, 0.0) # - part.relations = [r1] part.predicted_relations = [r2] evals = evaluator.evaluate(dataset) evaluation = evals(STUB_R_ID_1) print(evaluation) self.assertEqual(evaluation.tp, 1) self.assertEqual(evaluation.fn, 0) self.assertEqual(evaluation.fp, 0) computation = evals(STUB_R_ID_1).compute(strictness="exact") self.assertEqual(computation.f_measure, 1.0) # - part.relations = [r2] part.predicted_relations = [r1] evals = evaluator.evaluate(dataset) evaluation = evals(STUB_R_ID_1) self.assertEqual(evaluation.tp, 0) self.assertEqual(evaluation.fn, 1) self.assertEqual(evaluation.fp, 1) computation = evals(STUB_R_ID_1).compute(strictness="exact") self.assertEqual(computation.f_measure, 0.0)
def test_DocumentLevelRelationEvaluator_normalized_entities(self): evaluator = DocumentLevelRelationEvaluator( rel_type=STUB_R_ID_1, entity_map_fun=DocumentLevelRelationEvaluator.COMMON_ENTITY_MAP_FUNS['normalized_fun']( {STUB_E_ID_1: 'n_1', STUB_E_ID_2: 'n_1'}, penalize_unknown_normalizations="no") ) dataset = Dataset() doc_1 = Document() part_1 = Part('_irrelevant_') dataset.documents['doc_1'] = doc_1 doc_1.parts['part_1'] = part_1 part_1.relations = [ Relation( STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "Tool", norms={"n_1": "1964"}), Entity(STUB_E_ID_2, 0, "Maynard", norms={"n_1": "1961"})), ] # - part_1.predicted_relations = [ Relation( # One without normalization, one with another different normalization STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "Tool"), Entity(STUB_E_ID_2, 0, "Maynard", norms={"n_x": "1961"})), Relation( # One with different normalization, one with another different normalization STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "Tool", norms={"n_1": "666"}), Entity(STUB_E_ID_2, 0, "Maynard", norms={"n_x": "1961"})), Relation( # Both with the correct normalization ids, but one has wrong normalization STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "Tool", norms={"n_1": "666"}), Entity(STUB_E_ID_2, 0, "Maynard", norms={"n_1": "1961"})), Relation( # Both with another different normalization STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "Tool", norms={"n_another_key": "1964"}), Entity(STUB_E_ID_2, 0, "Maynard", norms={"n_another_key": "1961"})), ] self._apply_pipeline(dataset) # --- evals = evaluator.evaluate(dataset) evaluation = evals(STUB_R_ID_1) self.assertEqual(evaluation.tp, 0) self.assertEqual(evaluation.fn, 1) self.assertEqual(evaluation.fp, 1) computation = evals(STUB_R_ID_1).compute(strictness="exact") self.assertEqual(computation.f_measure, 0.0) # - part_1.predicted_relations = [ Relation( STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "Tool band", norms={"n_1": "1964"}), Entity(STUB_E_ID_2, 0, "Maynard James Keenan", norms={"n_1": "1961"})), ] evals = evaluator.evaluate(dataset) evaluation = evals(STUB_R_ID_1) self.assertEqual(evaluation.tp, 1) self.assertEqual(evaluation.fn, 0) self.assertEqual(evaluation.fp, 0) computation = evals(STUB_R_ID_1).compute(strictness="exact") self.assertEqual(computation.f_measure, 1.0)