def test_compute_ambiguity(self):
ann_1 = Annotations(self.ann_path_1)
ann_1_copy = Annotations(self.ann_path_1)
ambiguity = ann_1.compute_ambiguity(ann_1_copy)
# The number of overlapping spans for the selected ann file is known to be 25
self.assertEqual(25, len(ambiguity))
# Manually introduce ambiguity by changing the name of an entity in the copy
first_tuple = ann_1_copy.annotations[0]
ann_1_copy.annotations[0] = ('different_name', first_tuple[1], first_tuple[2], first_tuple[3])
ambiguity = ann_1.compute_ambiguity(ann_1_copy)
# See if this increased the ambiguity score by one
self.assertEqual(26, len(ambiguity))
def compute_ambiguity(self, dataset):
"""
Finds occurrences of spans from 'dataset' that intersect with a span from this annotation but do not have this spans label.
label. If 'dataset' comprises a models predictions, this method provides a strong indicators
of a model's in-ability to dis-ambiguate between entities. For a full analysis, compute a confusion matrix.
:param dataset: a Dataset object containing a predicted version of this dataset.
:param leniency: a floating point value between [0,1] defining the leniency of the character spans to count as different. A value of zero considers only exact character matches while a positive value considers entities that differ by up to :code:`ceil(leniency * len(span)/2)` on either side.
:return: a dictionary containing the ambiguity computations on each gold, predicted file pair
"""
if not isinstance(dataset, Dataset):
raise ValueError("dataset must be instance of Dataset")
# verify files are consistent
diff = set(file.ann_path.split(os.sep)[-1] for file in self) - set(file.ann_path.split(os.sep)[-1] for file in dataset)
if diff:
raise ValueError("Dataset of predictions is missing the files: " + str(list(diff)))
#Dictionary storing ambiguity over dataset
ambiguity_dict = {}
for gold_data_file in self:
prediction_iter = iter(dataset)
prediction_data_file = next(prediction_iter)
while str(gold_data_file) != str(prediction_data_file):
prediction_data_file = next(prediction_iter)
gold_annotation = Annotations(gold_data_file.ann_path)
pred_annotation = Annotations(prediction_data_file.ann_path)
# compute matrix on the Annotation file level
ambiguity_dict[str(gold_data_file)] = gold_annotation.compute_ambiguity(pred_annotation)
return ambiguity_dict
def compute_ambiguity(self, dataset):
"""
Finds occurrences of spans from 'dataset' that intersect with a span from this annotation but do not have this spans label.
label. If 'dataset' comprises a models predictions, this method provides a strong indicators
of a model's in-ability to dis-ambiguate between entities. For a full analysis, compute a confusion matrix.
:param dataset: a Dataset object containing a predicted version of this dataset.
:return: a dictionary containing the ambiguity computations on each gold, predicted file pair
"""
if not isinstance(dataset, Dataset):
raise ValueError("dataset must be instance of Dataset")
# verify files are consistent
diff = {d.file_name for d in self} - {d.file_name for d in dataset}
if diff:
raise ValueError(
f"Dataset of predictions is missing the files: {repr(diff)}")
# Dictionary storing ambiguity over dataset
ambiguity_dict = {}
for gold_data_file in self:
prediction_iter = iter(dataset)
prediction_data_file = next(prediction_iter)
while str(gold_data_file) != str(prediction_data_file):
prediction_data_file = next(prediction_iter)
gold_annotation = Annotations(gold_data_file.ann_path)
pred_annotation = Annotations(prediction_data_file.ann_path)
# compute matrix on the Annotation file level
ambiguity_dict[str(
gold_data_file)] = gold_annotation.compute_ambiguity(
pred_annotation)
return ambiguity_dict
