def test_DocumentLevelRelationEvaluator_parts_irrelevant(self):
evaluator = DocumentLevelRelationEvaluator(rel_type=STUB_R_ID_1)
dataset = Dataset()
doc_1 = Document()
part_1 = Part('_irrelevant_ PART *1*')
dataset.documents['doc_1'] = doc_1
doc_1.parts['part_1'] = part_1
part_2 = Part('_irrelevant_ PART *2*')
dataset.documents['doc_1'] = doc_1
doc_1.parts['part_2'] = part_2
part_1.relations = [
Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "TOOL"), Entity(STUB_E_ID_2, 0, "maynard")),
]
# -
part_2.predicted_relations = [
Relation(STUB_R_ID_1, Entity(STUB_E_ID_2, 0, "maynard"), Entity(STUB_E_ID_1, 0, "TOOL")),
]
self._apply_pipeline(dataset)
# ---
evals = evaluator.evaluate(dataset)
evaluation = evals(STUB_R_ID_1)
self.assertEqual(evaluation.tp, 1)
self.assertEqual(evaluation.fn, 0)
self.assertEqual(evaluation.fp, 0)
computation = evals(STUB_R_ID_1).compute(strictness="exact")
self.assertEqual(computation.f_measure, 1.0)
def test_DocumentLevelRelationEvaluator_default_entities_case_irrelevant(
self):
evaluator = DocumentLevelRelationEvaluator(rel_type=STUB_R_ID_1)
dataset = Dataset()
doc_1 = Document()
part_1 = Part('_irrelevant_')
dataset.documents['doc_1'] = doc_1
doc_1.parts['part_1'] = part_1
part_1.relations = [
Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "TOOL"),
Entity(STUB_E_ID_2, 0, "maynard")),
]
# -
part_1.predicted_relations = [
# empty
]
self._apply_pipeline(dataset)
# -
evals = evaluator.evaluate(dataset)
evaluation = evals(STUB_R_ID_1)
self.assertEqual(evaluation.tp, 0)
computation = evals(STUB_R_ID_1).compute(strictness="exact")
self.assertEqual(computation.f_measure, 0.0)
# ---
part_1.predicted_relations = [
Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "TOOL"),
Entity(STUB_E_ID_2, 0, "maynard")),
]
evals = evaluator.evaluate(dataset)
evaluation = evals(STUB_R_ID_1)
self.assertEqual(evaluation.tp, 1)
computation = evals(STUB_R_ID_1).compute(strictness="exact")
self.assertEqual(computation.f_measure, 1.0)
# -
part_1.predicted_relations = [
Relation(STUB_R_ID_1, Entity(STUB_E_ID_1, 0, "tool"),
Entity(STUB_E_ID_2, 0, "MAYNARD")),
]
evals = evaluator.evaluate(dataset)
evaluation = evals(STUB_R_ID_1)
self.assertEqual(evaluation.tp, 1)
computation = evals(STUB_R_ID_1).compute(strictness="exact")
self.assertEqual(computation.f_measure, 1.0)
def test_DocumentLevelRelationEvaluator_normalized_entities(self):
evaluator = DocumentLevelRelationEvaluator(
rel_type=STUB_R_ID_1,
entity_map_fun=DocumentLevelRelationEvaluator.COMMON_ENTITY_MAP_FUNS['normalized_fun'](
{STUB_E_ID_1: 'n_1', STUB_E_ID_2: 'n_1'},
penalize_unknown_normalizations="no")
)
dataset = Dataset()
doc_1 = Document()
part_1 = Part('_irrelevant_')
dataset.documents['doc_1'] = doc_1
doc_1.parts['part_1'] = part_1
part_1.relations = [
Relation(
STUB_R_ID_1,
Entity(STUB_E_ID_1, 0, "Tool", norms={"n_1": "1964"}),
Entity(STUB_E_ID_2, 0, "Maynard", norms={"n_1": "1961"})),
]
# -
part_1.predicted_relations = [
Relation(
# One without normalization, one with another different normalization
STUB_R_ID_1,
Entity(STUB_E_ID_1, 0, "Tool"),
Entity(STUB_E_ID_2, 0, "Maynard", norms={"n_x": "1961"})),
Relation(
# One with different normalization, one with another different normalization
STUB_R_ID_1,
Entity(STUB_E_ID_1, 0, "Tool", norms={"n_1": "666"}),
Entity(STUB_E_ID_2, 0, "Maynard", norms={"n_x": "1961"})),
Relation(
# Both with the correct normalization ids, but one has wrong normalization
STUB_R_ID_1,
Entity(STUB_E_ID_1, 0, "Tool", norms={"n_1": "666"}),
Entity(STUB_E_ID_2, 0, "Maynard", norms={"n_1": "1961"})),
Relation(
# Both with another different normalization
STUB_R_ID_1,
Entity(STUB_E_ID_1, 0, "Tool", norms={"n_another_key": "1964"}),
Entity(STUB_E_ID_2, 0, "Maynard", norms={"n_another_key": "1961"})),
]
self._apply_pipeline(dataset)
# ---
evals = evaluator.evaluate(dataset)
evaluation = evals(STUB_R_ID_1)
self.assertEqual(evaluation.tp, 0)
self.assertEqual(evaluation.fn, 1)
self.assertEqual(evaluation.fp, 1)
computation = evals(STUB_R_ID_1).compute(strictness="exact")
self.assertEqual(computation.f_measure, 0.0)
# -
part_1.predicted_relations = [
Relation(
STUB_R_ID_1,
Entity(STUB_E_ID_1, 0, "Tool band", norms={"n_1": "1964"}),
Entity(STUB_E_ID_2, 0, "Maynard James Keenan", norms={"n_1": "1961"})),
]
evals = evaluator.evaluate(dataset)
evaluation = evals(STUB_R_ID_1)
self.assertEqual(evaluation.tp, 1)
self.assertEqual(evaluation.fn, 0)
self.assertEqual(evaluation.fp, 0)
computation = evals(STUB_R_ID_1).compute(strictness="exact")
self.assertEqual(computation.f_measure, 1.0)