def test_ordering_both_ref_none(self): pair_a = alignment.LabelPair( None, annotations.Label('a2', start=1.66, end=1.92)) pair_b = alignment.LabelPair( None, annotations.Label('b2', start=1.65, end=1.92)) assert pair_b < pair_a
def test_ordering(self): pair_a = alignment.LabelPair( annotations.Label('a1', start=1.55, end=1.88), annotations.Label('a2', start=1.66, end=1.92)) pair_b = alignment.LabelPair( annotations.Label('b1', start=1.59, end=1.88), annotations.Label('b2', start=1.66, end=1.92)) assert pair_a < pair_b
def test_ordering_only_end_differs(self): pair_a = alignment.LabelPair( annotations.Label('a1', start=1.55, end=1.88), annotations.Label('a2', start=1.66, end=1.92)) pair_b = alignment.LabelPair( annotations.Label('b1', start=1.55, end=1.87), annotations.Label('b2', start=1.66, end=1.92)) assert pair_b < pair_a
def test_align(self, kws_ref_and_hyp_label_list): ll_ref, ll_hyp = kws_ref_and_hyp_label_list aligner = alignment.BipartiteMatchingAligner( substitution_penalty=2, non_overlap_penalty_weight=1 ) matches = aligner.align(ll_ref.labels, ll_hyp.labels) expected_matches = [ alignment.LabelPair(annotations.Label('up', start=5.28, end=5.99), annotations.Label('up', start=5.20, end=5.88)), alignment.LabelPair(annotations.Label('down', start=10.35, end=11.12), annotations.Label('right', start=10.30, end=11.08)), alignment.LabelPair(annotations.Label('right', start=20.87, end=22.01), None), alignment.LabelPair(annotations.Label('up', start=33.00, end=33.4), None), alignment.LabelPair(annotations.Label('up', start=33.4, end=33.8), None), alignment.LabelPair(annotations.Label('down', start=39.28, end=40.0), annotations.Label('down', start=39.27, end=40.01)), alignment.LabelPair(None, annotations.Label('up', start=32.00, end=32.5)), alignment.LabelPair(None, annotations.Label('up', start=34.2, end=34.8)), alignment.LabelPair(None, annotations.Label('left', start=39.3, end=39.9)) ] assert sorted(expected_matches) == sorted(matches)
def test_evaluate_with_two_label_lists(self, kws_ref_and_hyp_label_list): ll_ref, ll_hyp = kws_ref_and_hyp_label_list result = evaluator.KWSEvaluator().evaluate(ll_ref, ll_hyp) expected_matches = [ alignment.LabelPair(annotations.Label('up', start=5.28, end=5.99), annotations.Label('up', start=5.20, end=5.88)), alignment.LabelPair( annotations.Label('down', start=10.35, end=11.12), annotations.Label('right', start=10.30, end=11.08)), alignment.LabelPair( annotations.Label('right', start=20.87, end=22.01), None), alignment.LabelPair(annotations.Label('up', start=33.00, end=33.4), None), alignment.LabelPair(annotations.Label('up', start=33.4, end=33.8), None), alignment.LabelPair( annotations.Label('down', start=39.28, end=40.0), annotations.Label('down', start=39.27, end=40.01)), alignment.LabelPair(None, annotations.Label('up', start=32.00, end=32.5)), alignment.LabelPair(None, annotations.Label('up', start=34.2, end=34.8)), alignment.LabelPair( None, annotations.Label('left', start=39.3, end=39.9)) ] assert isinstance(result, evaluator.KWSEvaluation) assert sorted(expected_matches) == sorted(result.label_pairs)
def test_align_empty_ref_returns_all_none(self): lev = alignment.LevenshteinAligner() ali = lev.align( ll_with_values([]), ll_with_values(['a', 'b', 'c']) ) assert ali == [ alignment.LabelPair(None, annotations.Label('a')), alignment.LabelPair(None, annotations.Label('b')), alignment.LabelPair(None, annotations.Label('c')), ]
def test_align_substitution(self): lev = alignment.LevenshteinAligner() ali = lev.align( ll_with_values(['a', 'b', 'c']), ll_with_values(['a', 'x', 'c']) ) assert ali == [ alignment.LabelPair(annotations.Label('a'), annotations.Label('a')), alignment.LabelPair(annotations.Label('b'), annotations.Label('x')), alignment.LabelPair(annotations.Label('c'), annotations.Label('c')), ]
def test_align_high_substitution_cost_forces_deletions_and_insertions(self): lev = alignment.LevenshteinAligner(substitution_cost=20) ali = lev.align( ll_with_values(['a', 'b', 'c']), ll_with_values(['a', 'x', 'c']) ) assert ali == [ alignment.LabelPair(annotations.Label('a'), annotations.Label('a')), alignment.LabelPair(annotations.Label('b'), None), alignment.LabelPair(None, annotations.Label('x')), alignment.LabelPair(annotations.Label('c'), annotations.Label('c')), ]
def test_create_from_label_pairs(): pairs = [ alignment.LabelPair(annotations.Label('up', start=5.28, end=5.99), annotations.Label('up', start=5.20, end=5.88)), alignment.LabelPair(annotations.Label('down', start=10.35, end=11.12), annotations.Label('right', start=10.30, end=11.08)), alignment.LabelPair(annotations.Label('right', start=20.87, end=22.01), None), alignment.LabelPair(annotations.Label('up', start=33.00, end=33.4), None), alignment.LabelPair(annotations.Label('up', start=33.4, end=33.8), None), alignment.LabelPair(annotations.Label('down', start=39.28, end=40.0), annotations.Label('down', start=39.27, end=40.01)), alignment.LabelPair(None, annotations.Label('up', start=32.00, end=32.5)), alignment.LabelPair(None, annotations.Label('up', start=34.2, end=34.8)), alignment.LabelPair(None, annotations.Label('left', start=39.3, end=39.9)) ] cnf = confusion.create_from_label_pairs(pairs) assert 'up' in cnf.instances.keys() assert 'down' in cnf.instances.keys() assert 'right' in cnf.instances.keys() assert 'left' in cnf.instances.keys() assert cnf.instances['up'].correct == 1 assert cnf.instances['up'].deletions == 2 assert cnf.instances['up'].insertions == 2 assert cnf.instances['up'].substitutions == 0 assert cnf.instances['up'].substitutions_out == 0 assert cnf.instances['down'].correct == 1 assert cnf.instances['down'].deletions == 0 assert cnf.instances['down'].insertions == 0 assert cnf.instances['down'].substitutions == 1 assert cnf.instances['down'].substitutions_out == 0 assert cnf.instances['right'].correct == 0 assert cnf.instances['right'].deletions == 1 assert cnf.instances['right'].insertions == 0 assert cnf.instances['right'].substitutions == 0 assert cnf.instances['right'].substitutions_out == 1 assert cnf.instances['left'].correct == 0 assert cnf.instances['left'].deletions == 0 assert cnf.instances['left'].insertions == 1 assert cnf.instances['left'].substitutions == 0 assert cnf.instances['left'].substitutions_out == 0
def test_evaluate(self): ref = evaluator.Outcome( label_lists={ 'a': annotations.LabelList( labels=[annotations.Label('a b a d f a b')]) }) hyp = evaluator.Outcome(label_lists={ 'a': annotations.LabelList(labels=[annotations.Label('a b d f i b')]) }) result = evaluator.ASREvaluator().do_evaluate(ref, hyp) assert len(result.utt_to_label_pairs) == 1 assert result.utt_to_label_pairs['a'] == [ alignment.LabelPair(annotations.Label('a'), annotations.Label('a')), alignment.LabelPair(annotations.Label('b'), annotations.Label('b')), alignment.LabelPair(annotations.Label('a'), None), alignment.LabelPair(annotations.Label('d'), annotations.Label('d')), alignment.LabelPair(annotations.Label('f'), annotations.Label('f')), alignment.LabelPair(annotations.Label('a'), annotations.Label('i')), alignment.LabelPair(annotations.Label('b'), annotations.Label('b')), ]
def test_evaluate_with_multiple_labels(self): ref = evaluator.Outcome( label_lists={ 'a': annotations.LabelList(labels=[ annotations.Label('a b', start=0, end=3), annotations.Label('a d', start=3, end=5), annotations.Label('f a b', start=5, end=6) ]) }) hyp = evaluator.Outcome(label_lists={ 'a': annotations.LabelList(labels=[annotations.Label('a b d f i b')]) }) result = evaluator.ASREvaluator().do_evaluate(ref, hyp) assert len(result.utt_to_label_pairs) == 1 assert result.utt_to_label_pairs['a'] == [ alignment.LabelPair(annotations.Label('a'), annotations.Label('a')), alignment.LabelPair(annotations.Label('b'), annotations.Label('b')), alignment.LabelPair(annotations.Label('a'), None), alignment.LabelPair(annotations.Label('d'), annotations.Label('d')), alignment.LabelPair(annotations.Label('f'), annotations.Label('f')), alignment.LabelPair(annotations.Label('a'), annotations.Label('i')), alignment.LabelPair(annotations.Label('b'), annotations.Label('b')), ]
def test_align_endless_labels(self): ref_ll = [ annotations.Label('a', 4.2, 8.5), annotations.Label('b', 13.1, 19.23) ] hyp_ll = [ annotations.Label('x', 9.2, float('inf')) ] result = alignment.FullMatchingAligner(0.1).align(ref_ll, hyp_ll) assert sorted(result) == sorted([ alignment.LabelPair(annotations.Label('a', 4.2, 8.5), None), alignment.LabelPair(annotations.Label('b', 13.1, 19.23), annotations.Label('x', 9.2, float('inf'))) ])
def test_align_deletion(self): ref_ll = [annotations.Label('a', 4.2, 8.5)] hyp_ll = [] result = alignment.FullMatchingAligner(0.1).align(ref_ll, hyp_ll) assert result == [ alignment.LabelPair(annotations.Label('a', 4.2, 8.5), None) ]
def test_align_insertion(self): ref_ll = [] hyp_ll = [annotations.Label('y', 7.6, 15.2)] result = alignment.FullMatchingAligner(0.1).align(ref_ll, hyp_ll) assert result == [ alignment.LabelPair(None, annotations.Label('y', 7.6, 15.2)) ]
def test_align(self): ref_ll = [ annotations.Label('a', 4.2, 8.5), annotations.Label('b', 13.1, 19.23) ] hyp_ll = [ annotations.Label('x', 3.2, 5.4), annotations.Label('y', 7.6, 15.2) ] result = alignment.FullMatchingAligner(0.1).align(ref_ll, hyp_ll) assert result == [ alignment.LabelPair(annotations.Label('a', 4.2, 8.5), annotations.Label('x', 3.2, 5.4)), alignment.LabelPair(annotations.Label('a', 4.2, 8.5), annotations.Label('y', 7.6, 15.2)), alignment.LabelPair(annotations.Label('b', 13.1, 19.23), annotations.Label('y', 7.6, 15.2)) ]
def test_align_empty_ref_returns_insertions(self): ll_ref = [] ll_hyp = [annotations.Label('greasy', 1.4, 1.9)] aligner = alignment.BipartiteMatchingAligner( substitution_penalty=2, non_overlap_penalty_weight=1 ) matches = aligner.align(ll_ref, ll_hyp) assert matches == [ alignment.LabelPair(None, annotations.Label('greasy', 1.4, 1.9)) ]
def test_evaluate_computes_correct_alignment(self, ll_ref, ll_hyp): aligner = alignment.BipartiteMatchingAligner() result = evaluator.EventEvaluator(aligner).evaluate(ll_ref, ll_hyp) expected_matches = [ alignment.LabelPair(annotations.Label('up', start=5.28, end=5.99), annotations.Label('up', start=5.20, end=5.88)), alignment.LabelPair(annotations.Label('down', start=10.35, end=11.12), annotations.Label('right', start=10.30, end=11.08)), alignment.LabelPair(annotations.Label('right', start=20.87, end=22.01), None), alignment.LabelPair(annotations.Label('up', start=33.00, end=33.4), None), alignment.LabelPair(annotations.Label('up', start=33.4, end=33.8), None), alignment.LabelPair(annotations.Label('down', start=39.28, end=40.0), annotations.Label('down', start=39.27, end=40.01)), alignment.LabelPair(None, annotations.Label('up', start=32.00, end=32.5)), alignment.LabelPair(None, annotations.Label('up', start=34.2, end=34.8)), alignment.LabelPair(None, annotations.Label('left', start=39.3, end=39.9)) ] assert isinstance(result, evaluator.EventEvaluation) assert sorted(expected_matches) == sorted(result.label_pairs)
def sample_confusion(): conf = confusion.EventConfusion('up') conf.correct_pairs.extend([ alignment.LabelPair(annotations.Label('up', start=5.28, end=5.99), annotations.Label('up', start=5.20, end=5.88)), alignment.LabelPair(annotations.Label('up', start=39.28, end=40.0), annotations.Label('down', start=39.27, end=40.01)) ]) conf.insertion_pairs.extend([ alignment.LabelPair(None, annotations.Label('up', start=32.00, end=32.5)), alignment.LabelPair(None, annotations.Label('up', start=34.2, end=34.8)), alignment.LabelPair(None, annotations.Label('up', start=55.3, end=56.9)) ]) conf.deletion_pairs.extend([ alignment.LabelPair(annotations.Label('up', start=20.87, end=22.01), None), alignment.LabelPair(annotations.Label('up', start=33.00, end=33.4), None), alignment.LabelPair(annotations.Label('up', start=33.4, end=33.8), None), ]) conf.substitution_pairs['right'].extend([ alignment.LabelPair(annotations.Label('up', start=10.35, end=11.12), annotations.Label('right', start=10.30, end=11.18)), ]) conf.substitution_pairs['down'].extend([ alignment.LabelPair(annotations.Label('up', start=74.28, end=75.0), annotations.Label('down', start=74.17, end=75.01)) ]) conf.substitution_out_pairs['left'].extend([ alignment.LabelPair(annotations.Label('left', start=15.35, end=16.12), annotations.Label('up', start=15.4, end=16.18)), ]) conf.substitution_out_pairs['up'].extend([ alignment.LabelPair(annotations.Label('down', start=84.28, end=85.09), annotations.Label('up', start=84.17, end=85.01)) ]) return conf