def test_confusion_matrix(self):
ann_1 = Annotations(self.ann_path_1)
ann_2 = Annotations(self.ann_path_2)
ann_1.add_entity(*ann_2.annotations[0])
self.assertEqual(
len(ann_1.compute_confusion_matrix(ann_2, self.entities)[0]),
len(self.entities))
self.assertEqual(
len(ann_1.compute_confusion_matrix(ann_2, self.entities)),
len(self.entities))
def compute_confusion_matrix(self, other, leniency=0):
"""
Generates a confusion matrix where this Dataset serves as the gold standard annotations and `dataset` serves
as the predicted annotations. A typical workflow would involve creating a Dataset object with the prediction directory
outputted by a model and then passing it into this method.
:param other: a Dataset object containing a predicted version of this dataset.
:param leniency: a floating point value between [0,1] defining the leniency of the character spans to count as different. A value of zero considers only exact character matches while a positive value considers entities that differ by up to :code:`ceil(leniency * len(span)/2)` on either side.
:return: two element tuple containing a label array (of entity names) and a matrix where rows are gold labels and columns are predicted labels. matrix[i][j] indicates that entities[i] in this dataset was predicted as entities[j] in 'annotation' matrix[i][j] times
"""
if not isinstance(other, Dataset):
raise ValueError("other must be instance of Dataset")
# verify files are consistent
diff = {d.file_name for d in self} - {d.file_name for d in other}
if diff:
raise ValueError(
f"Dataset of predictions is missing the files: {repr(diff)}")
# sort entities in ascending order by count.
entities = [
key for key, _ in sorted(self.compute_counts().items(),
key=lambda x: x[1])
]
confusion_matrix = [[0 * len(entities)] * len(entities)]
for gold_data_file in self:
prediction_iter = iter(other)
prediction_data_file = next(prediction_iter)
while str(gold_data_file) != str(prediction_data_file):
prediction_data_file = next(prediction_iter)
gold_annotation = Annotations(gold_data_file.ann_path)
pred_annotation = Annotations(prediction_data_file.ann_path)
# compute matrix on the Annotation file level
ann_confusion_matrix = gold_annotation.compute_confusion_matrix(
pred_annotation, entities, leniency=leniency)
for i in range(len(confusion_matrix)):
for j in range(len(confusion_matrix)):
confusion_matrix[i][j] += ann_confusion_matrix[i][j]
return entities, confusion_matrix
