def test_two_summaries_end_inserting(self):
partial_1 = ErrorsSummary.from_lists(list('ab'), list('acd'))
partial_2 = ErrorsSummary.from_lists(list('ab'), list('abc'))
summary = ErrorsSummary.aggregate([partial_1, partial_2])
self.assertEqual(summary.nb_errors, 3)
self.assertEqual(summary.nb_subs, 1)
self.assertEqual(summary.nb_inss, 2)
self.assertEqual(summary.nb_dels, 0)
self.assertEqual(summary.ref_len, 4)
self.assertEqual(summary.nb_lines_summarized, 2)
self.assertEqual(summary.confusions['a']['a'], 2)
self.assertEqual(summary.confusions['b']['b'], 1)
unmatched_b_correctly_matched_to_c = (summary.confusions['b']['c'] == 1
and summary.confusions[None]['d']
== 1)
unmatched_b_correctly_matched_to_d = (
summary.confusions[None]['c'] == 1
and summary.confusions['b']['d'] == 1)
self.assertTrue(unmatched_b_correctly_matched_to_c
or unmatched_b_correctly_matched_to_d)
self.assertEqual(
sum(sum(k.values()) for k in summary.confusions.values()),
len('acd') + len('abc'))
self.assertEqual(summary.ending_errors.pure_deletions, 0)
self.assertEqual(summary.ending_errors.mixed_deletions, 0)
self.assertEqual(summary.ending_errors.pure_insertions, 1)
self.assertEqual(summary.ending_errors.mixed_insertions, 1)
self.assertEqual(summary.ending_errors.correct, 0)
def test_one_errorneuous_one_perfect(self):
partial_1 = ErrorsSummary.from_lists(list('abcd'), list('abb'))
partial_2 = ErrorsSummary.from_lists(list('ab'), list('ab'))
summary = ErrorsSummary.aggregate([partial_1, partial_2])
self.assertEqual(summary.nb_errors, 2)
self.assertEqual(summary.nb_subs, 1)
self.assertEqual(summary.nb_inss, 0)
self.assertEqual(summary.nb_dels, 1)
self.assertEqual(summary.ref_len, 6)
self.assertEqual(summary.nb_lines_summarized, 2)
self.assertEqual(summary.confusions['a']['a'], 2)
self.assertEqual(summary.confusions['b']['a'], 0)
self.assertEqual(summary.confusions['b']['b'], 2)
extra_b_correctly_matched_to_c = (summary.confusions['c']['b'] == 1 and
summary.confusions['d'][None] == 1)
extra_b_correctly_matched_to_d = (summary.confusions['c'][None] == 1
and summary.confusions['d']['b']
== 1)
self.assertTrue(extra_b_correctly_matched_to_c
or extra_b_correctly_matched_to_d)
self.assertEqual(
sum(sum(k.values()) for k in summary.confusions.values()),
len('abcd') + len('ab'))
self.assertEqual(summary.ending_errors.pure_deletions, 0)
self.assertEqual(summary.ending_errors.mixed_deletions, 1)
self.assertEqual(summary.ending_errors.correct, 1)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--ground-truth', type=str, required=True)
parser.add_argument('--transcriptions', type=str, required=True)
parser.add_argument('--output', type=str, required=False, default=None)
parser.add_argument('--char-confusion',
type=str,
required=False,
default=None)
parser.add_argument('--word-confusion',
type=str,
required=False,
default=None)
parser.add_argument(
'--human-readable',
action='store_true',
required=False,
help='make per-line output easier to process by humans')
args = parser.parse_args()
ground_truths = load_transcriptions(args.ground_truth)
transcriptions = load_transcriptions(args.transcriptions)
lines, all_matched = match_lines(transcriptions, ground_truths)
char_summary = ErrorsSummary.aggregate(
[line.char_errors for line in lines])
word_summary = ErrorsSummary.aggregate([line.word_error for line in lines])
print("CER:", char_summary, '' if all_matched else "[PARTIAL]")
print("WER:", word_summary, '' if all_matched else "[PARTIAL]")
if args.output is not None:
save(sort(lines), args.output, args.human_readable)
if args.char_confusion is not None:
save_confusions(char_summary, args.char_confusion)
if args.word_confusion is not None:
save_confusions(word_summary, args.word_confusion)
return 0
def test_one_substituting_one_perfect(self):
partial_1 = ErrorsSummary.from_lists(list('abcd'), list('abxy'))
partial_2 = ErrorsSummary.from_lists(list('ab'), list('ab'))
summary = ErrorsSummary.aggregate([partial_1, partial_2])
self.assertEqual(summary.nb_errors, 2)
self.assertEqual(summary.nb_subs, 2)
self.assertEqual(summary.nb_inss, 0)
self.assertEqual(summary.nb_dels, 0)
self.assertEqual(summary.ref_len, 6)
self.assertEqual(summary.nb_lines_summarized, 2)
self.assertEqual(summary.confusions['a']['a'], 2)
self.assertEqual(summary.confusions['b']['b'], 2)
self.assertEqual(summary.confusions['c']['x'], 1)
self.assertEqual(summary.confusions['d']['y'], 1)
self.assertEqual(
sum(sum(k.values()) for k in summary.confusions.values()),
len('abcd') + len('ab'))
self.assertEqual(summary.ending_errors.pure_deletions, 0)
self.assertEqual(summary.ending_errors.mixed_deletions, 0)
self.assertEqual(summary.ending_errors.pure_substitutions, 1)
self.assertEqual(summary.ending_errors.correct, 1)