def test_DocumentLevelRelationEvaluator_repeated_relations_irrelevant( self): evaluator = DocumentLevelRelationEvaluator(rel_type=STUB_R_ID_1) dataset = Dataset() doc_1 = Document() part_1 = Part('_irrelevant_') dataset.documents['doc_1'] = doc_1 doc_1.parts['part_1'] = part_1 part_1.relations = [ Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "TOOL"), Entity(STUB_E_ID_2, 0, "maynard")), Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "TOOL"), Entity(STUB_E_ID_2, 0, "Danny Carey")), Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 1, "TOOL"), Entity(STUB_E_ID_2, 1, "Danny Carey")), ] # - part_1.predicted_relations = [ Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "TOOL"), Entity(STUB_E_ID_2, 0, "maynard")), Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 1, "TOOL"), Entity(STUB_E_ID_2, 1, "maynard")), ] self._apply_pipeline(dataset) # --- evals = evaluator.evaluate(dataset) evaluation = evals(STUB_R_ID_1) self.assertEqual(evaluation.tp, 1) self.assertEqual(evaluation.fn, 1) self.assertEqual(evaluation.fp, 0) computation = evals(STUB_R_ID_1).compute(strictness="exact") self.assertEqual(computation.f_measure, 0.6666666666666666) # - part_1.predicted_relations = [ Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 2, "TOOL"), Entity(STUB_E_ID_2, 2, "maynard")), Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 3, "TOOL"), Entity(STUB_E_ID_2, 3, "maynard")), Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 4, "TOOL"), Entity(STUB_E_ID_2, 4, "Danny Carey")), ] evals = evaluator.evaluate(dataset) evaluation = evals(STUB_R_ID_1) self.assertEqual(evaluation.tp, 2) self.assertEqual(evaluation.fn, 0) self.assertEqual(evaluation.fp, 0) computation = evals(STUB_R_ID_1).compute(strictness="exact") self.assertEqual(computation.f_measure, 1.0)
def test_DocumentLevelRelationEvaluator_default_entities_case_irrelevant( self): evaluator = DocumentLevelRelationEvaluator(rel_type=STUB_R_ID_1) dataset = Dataset() doc_1 = Document() part_1 = Part('_irrelevant_') dataset.documents['doc_1'] = doc_1 doc_1.parts['part_1'] = part_1 part_1.relations = [ Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "TOOL"), Entity(STUB_E_ID_2, 0, "maynard")), ] # - part_1.predicted_relations = [ # empty ] self._apply_pipeline(dataset) # - evals = evaluator.evaluate(dataset) evaluation = evals(STUB_R_ID_1) self.assertEqual(evaluation.tp, 0) computation = evals(STUB_R_ID_1).compute(strictness="exact") self.assertEqual(computation.f_measure, 0.0) # --- part_1.predicted_relations = [ Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "TOOL"), Entity(STUB_E_ID_2, 0, "maynard")), ] evals = evaluator.evaluate(dataset) evaluation = evals(STUB_R_ID_1) self.assertEqual(evaluation.tp, 1) computation = evals(STUB_R_ID_1).compute(strictness="exact") self.assertEqual(computation.f_measure, 1.0) # - part_1.predicted_relations = [ Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "tool"), Entity(STUB_E_ID_2, 0, "MAYNARD")), ] evals = evaluator.evaluate(dataset) evaluation = evals(STUB_R_ID_1) self.assertEqual(evaluation.tp, 1) computation = evals(STUB_R_ID_1).compute(strictness="exact") self.assertEqual(computation.f_measure, 1.0)
def test_DocumentLevelRelationEvaluator_false_positives(self): evaluator = DocumentLevelRelationEvaluator(rel_type=STUB_R_ID_1) dataset = Dataset() doc_1 = Document() part_1 = Part('_irrelevant_ PART *1*') dataset.documents['doc_1'] = doc_1 doc_1.parts['part_1'] = part_1 part_2 = Part('_irrelevant_ PART *2*') dataset.documents['doc_1'] = doc_1 doc_1.parts['part_2'] = part_2 part_1.relations = [ Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "TOOL"), Entity(STUB_E_ID_2, 0, "Maynard")), ] # - part_2.predicted_relations = [ Relation(STUB_R_ID_1, Entity(STUB_E_ID_2, 0, "TOOL"), Entity(STUB_E_ID_1, 0, "Snoop Dog")), ] self._apply_pipeline(dataset) # --- evals = evaluator.evaluate(dataset) evaluation = evals(STUB_R_ID_1) self.assertEqual(evaluation.tp, 0) self.assertEqual(evaluation.fn, 1) self.assertEqual(evaluation.fp, 1) computation = evals(STUB_R_ID_1).compute(strictness="exact") self.assertEqual(computation.f_measure, 0.0)
def test_DocumentLevelRelationEvaluator_normalized_entities(self): evaluator = DocumentLevelRelationEvaluator( rel_type=STUB_R_ID_1, entity_map_fun=DocumentLevelRelationEvaluator.COMMON_ENTITY_MAP_FUNS['normalized_fun']( {STUB_E_ID_1: 'n_1', STUB_E_ID_2: 'n_1'}, penalize_unknown_normalizations="no") ) dataset = Dataset() doc_1 = Document() part_1 = Part('_irrelevant_') dataset.documents['doc_1'] = doc_1 doc_1.parts['part_1'] = part_1 part_1.relations = [ Relation( STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "Tool", norms={"n_1": "1964"}), Entity(STUB_E_ID_2, 0, "Maynard", norms={"n_1": "1961"})), ] # - part_1.predicted_relations = [ Relation( # One without normalization, one with another different normalization STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "Tool"), Entity(STUB_E_ID_2, 0, "Maynard", norms={"n_x": "1961"})), Relation( # One with different normalization, one with another different normalization STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "Tool", norms={"n_1": "666"}), Entity(STUB_E_ID_2, 0, "Maynard", norms={"n_x": "1961"})), Relation( # Both with the correct normalization ids, but one has wrong normalization STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "Tool", norms={"n_1": "666"}), Entity(STUB_E_ID_2, 0, "Maynard", norms={"n_1": "1961"})), Relation( # Both with another different normalization STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "Tool", norms={"n_another_key": "1964"}), Entity(STUB_E_ID_2, 0, "Maynard", norms={"n_another_key": "1961"})), ] self._apply_pipeline(dataset) # --- evals = evaluator.evaluate(dataset) evaluation = evals(STUB_R_ID_1) self.assertEqual(evaluation.tp, 0) self.assertEqual(evaluation.fn, 1) self.assertEqual(evaluation.fp, 1) computation = evals(STUB_R_ID_1).compute(strictness="exact") self.assertEqual(computation.f_measure, 0.0) # - part_1.predicted_relations = [ Relation( STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "Tool band", norms={"n_1": "1964"}), Entity(STUB_E_ID_2, 0, "Maynard James Keenan", norms={"n_1": "1961"})), ] evals = evaluator.evaluate(dataset) evaluation = evals(STUB_R_ID_1) self.assertEqual(evaluation.tp, 1) self.assertEqual(evaluation.fn, 0) self.assertEqual(evaluation.fp, 0) computation = evals(STUB_R_ID_1).compute(strictness="exact") self.assertEqual(computation.f_measure, 1.0)