def test_double_deletion(self): a = "a b c".split() b = "a".split() expected = [ (['a', 'b', 'c'], ['a']), ] self.assertEqual(align(a, b), expected)
def test_single_deletion_reversed(self): a = "a b".split() b = "b".split() expected = [ (['a', 'b'], ['b']), ] self.assertEqual(align(a, b), expected)
def test_double_insertion_reversed(self): a = "b".split() b = "c a b".split() expected = [ (['b'], ['c', 'a', 'b']), ] self.assertEqual(align(a, b), expected)
def test_substitution_only(self): a = "a".split() b = "b".split() expected = [ (['a'], ['b']), ] self.assertEqual(align(a, b), expected)
def test_single_insertion(self): a = "a".split() b = "a b".split() expected = [ (['a'], ['a', 'b']), ] self.assertEqual(align(a, b), expected)
def test_single_element(self): a = "a".split() b = "a".split() expected = [ (['a'], ['a']), ] self.assertEqual(align(a, b), expected)
def test_trivial(self): a = "a b".split() b = "a b".split() expected = [ (['a'], ['a']), (['b'], ['b']), ] self.assertEqual(align(a, b), expected)
def test_inversion_at_end(self): a = "a b c d".split() b = "a b d c".split() expected = [ (['a'], ['a']), (['b'], ['b']), (['c'], ['d']), (['d'], ['c']), ] self.assertEqual(align(a, b), expected)
def test_double_substituion(self): a = "a b c d".split() b = "a x y d".split() expected = [ (['a'], ['a']), (['b'], ['x']), (['c'], ['y']), (['d'], ['d']), ] self.assertEqual(align(a, b), expected)
def test_inner_insertion(self): a = "a d".split() b = "a b c d".split() expected_1 = [ (['a'], ['a', 'b', 'c']), (['d'], ['d']), ] expected_2 = [ (['a'], ['a']), (['d'], ['b', 'c', 'd']), ] self.assertIn(align(a, b), [expected_1, expected_2])
oov_hits: Dict[int, List[Tuple[str, List[str]]]] = {} references = {} with open(args.text_references) as f: for line in f: fields = line.split() references[fields[0]] = fields[1:] candidate_possible_words = [] for line in sys.stdin: fields = line.split() _, utt_id, _, _, _ = parse_oov_id(fields[0]) candidate_line = fields[1:] reference_line = references[utt_id] alignment = align(reference_line, candidate_line) mismatches = extract_mismatch(alignment) oov_mismatch = find_in_mismatches(mismatches, "<UNK-OI>") total_ref_len += len(reference_line) total_nb_errors += number_of_errors(mismatches) matching_oovs = intersection(oov_list, oov_mismatch[0]) if len(matching_oovs) in oov_hits: oov_hits[len(matching_oovs)].append((utt_id, matching_oovs)) else: oov_hits[len(matching_oovs)] = [(utt_id, matching_oovs)] candidate_possible_words.append(oov_mismatch[0]) print(fields[0], oov_mismatch[0], '--', oov_mismatch[1])