def test_ordering_both_ref_none(self):
pair_a = alignment.LabelPair(
None, annotations.Label('a2', start=1.66, end=1.92))
pair_b = alignment.LabelPair(
None, annotations.Label('b2', start=1.65, end=1.92))
assert pair_b < pair_a
def test_ordering(self):
pair_a = alignment.LabelPair(
annotations.Label('a1', start=1.55, end=1.88),
annotations.Label('a2', start=1.66, end=1.92))
pair_b = alignment.LabelPair(
annotations.Label('b1', start=1.59, end=1.88),
annotations.Label('b2', start=1.66, end=1.92))
assert pair_a < pair_b
def test_ordering_only_end_differs(self):
pair_a = alignment.LabelPair(
annotations.Label('a1', start=1.55, end=1.88),
annotations.Label('a2', start=1.66, end=1.92))
pair_b = alignment.LabelPair(
annotations.Label('b1', start=1.55, end=1.87),
annotations.Label('b2', start=1.66, end=1.92))
assert pair_b < pair_a
def test_align(self, kws_ref_and_hyp_label_list):
ll_ref, ll_hyp = kws_ref_and_hyp_label_list
aligner = alignment.BipartiteMatchingAligner(
substitution_penalty=2,
non_overlap_penalty_weight=1
)
matches = aligner.align(ll_ref.labels, ll_hyp.labels)
expected_matches = [
alignment.LabelPair(annotations.Label('up', start=5.28, end=5.99),
annotations.Label('up', start=5.20, end=5.88)),
alignment.LabelPair(annotations.Label('down', start=10.35, end=11.12),
annotations.Label('right', start=10.30, end=11.08)),
alignment.LabelPair(annotations.Label('right', start=20.87, end=22.01), None),
alignment.LabelPair(annotations.Label('up', start=33.00, end=33.4), None),
alignment.LabelPair(annotations.Label('up', start=33.4, end=33.8), None),
alignment.LabelPair(annotations.Label('down', start=39.28, end=40.0),
annotations.Label('down', start=39.27, end=40.01)),
alignment.LabelPair(None, annotations.Label('up', start=32.00, end=32.5)),
alignment.LabelPair(None, annotations.Label('up', start=34.2, end=34.8)),
alignment.LabelPair(None, annotations.Label('left', start=39.3, end=39.9))
]
assert sorted(expected_matches) == sorted(matches)
def test_evaluate_with_two_label_lists(self, kws_ref_and_hyp_label_list):
ll_ref, ll_hyp = kws_ref_and_hyp_label_list
result = evaluator.KWSEvaluator().evaluate(ll_ref, ll_hyp)
expected_matches = [
alignment.LabelPair(annotations.Label('up', start=5.28, end=5.99),
annotations.Label('up', start=5.20, end=5.88)),
alignment.LabelPair(
annotations.Label('down', start=10.35, end=11.12),
annotations.Label('right', start=10.30, end=11.08)),
alignment.LabelPair(
annotations.Label('right', start=20.87, end=22.01), None),
alignment.LabelPair(annotations.Label('up', start=33.00, end=33.4),
None),
alignment.LabelPair(annotations.Label('up', start=33.4, end=33.8),
None),
alignment.LabelPair(
annotations.Label('down', start=39.28, end=40.0),
annotations.Label('down', start=39.27, end=40.01)),
alignment.LabelPair(None,
annotations.Label('up', start=32.00,
end=32.5)),
alignment.LabelPair(None,
annotations.Label('up', start=34.2, end=34.8)),
alignment.LabelPair(
None, annotations.Label('left', start=39.3, end=39.9))
]
assert isinstance(result, evaluator.KWSEvaluation)
assert sorted(expected_matches) == sorted(result.label_pairs)
def test_align_empty_ref_returns_all_none(self):
lev = alignment.LevenshteinAligner()
ali = lev.align(
ll_with_values([]),
ll_with_values(['a', 'b', 'c'])
)
assert ali == [
alignment.LabelPair(None, annotations.Label('a')),
alignment.LabelPair(None, annotations.Label('b')),
alignment.LabelPair(None, annotations.Label('c')),
]
def test_align_substitution(self):
lev = alignment.LevenshteinAligner()
ali = lev.align(
ll_with_values(['a', 'b', 'c']),
ll_with_values(['a', 'x', 'c'])
)
assert ali == [
alignment.LabelPair(annotations.Label('a'), annotations.Label('a')),
alignment.LabelPair(annotations.Label('b'), annotations.Label('x')),
alignment.LabelPair(annotations.Label('c'), annotations.Label('c')),
]
def test_align_high_substitution_cost_forces_deletions_and_insertions(self):
lev = alignment.LevenshteinAligner(substitution_cost=20)
ali = lev.align(
ll_with_values(['a', 'b', 'c']),
ll_with_values(['a', 'x', 'c'])
)
assert ali == [
alignment.LabelPair(annotations.Label('a'), annotations.Label('a')),
alignment.LabelPair(annotations.Label('b'), None),
alignment.LabelPair(None, annotations.Label('x')),
alignment.LabelPair(annotations.Label('c'), annotations.Label('c')),
]
def test_create_from_label_pairs():
pairs = [
alignment.LabelPair(annotations.Label('up', start=5.28, end=5.99),
annotations.Label('up', start=5.20, end=5.88)),
alignment.LabelPair(annotations.Label('down', start=10.35, end=11.12),
annotations.Label('right', start=10.30,
end=11.08)),
alignment.LabelPair(annotations.Label('right', start=20.87, end=22.01),
None),
alignment.LabelPair(annotations.Label('up', start=33.00, end=33.4),
None),
alignment.LabelPair(annotations.Label('up', start=33.4, end=33.8),
None),
alignment.LabelPair(annotations.Label('down', start=39.28, end=40.0),
annotations.Label('down', start=39.27, end=40.01)),
alignment.LabelPair(None, annotations.Label('up',
start=32.00,
end=32.5)),
alignment.LabelPair(None, annotations.Label('up', start=34.2,
end=34.8)),
alignment.LabelPair(None,
annotations.Label('left', start=39.3, end=39.9))
]
cnf = confusion.create_from_label_pairs(pairs)
assert 'up' in cnf.instances.keys()
assert 'down' in cnf.instances.keys()
assert 'right' in cnf.instances.keys()
assert 'left' in cnf.instances.keys()
assert cnf.instances['up'].correct == 1
assert cnf.instances['up'].deletions == 2
assert cnf.instances['up'].insertions == 2
assert cnf.instances['up'].substitutions == 0
assert cnf.instances['up'].substitutions_out == 0
assert cnf.instances['down'].correct == 1
assert cnf.instances['down'].deletions == 0
assert cnf.instances['down'].insertions == 0
assert cnf.instances['down'].substitutions == 1
assert cnf.instances['down'].substitutions_out == 0
assert cnf.instances['right'].correct == 0
assert cnf.instances['right'].deletions == 1
assert cnf.instances['right'].insertions == 0
assert cnf.instances['right'].substitutions == 0
assert cnf.instances['right'].substitutions_out == 1
assert cnf.instances['left'].correct == 0
assert cnf.instances['left'].deletions == 0
assert cnf.instances['left'].insertions == 1
assert cnf.instances['left'].substitutions == 0
assert cnf.instances['left'].substitutions_out == 0
def test_evaluate(self):
ref = evaluator.Outcome(
label_lists={
'a':
annotations.LabelList(
labels=[annotations.Label('a b a d f a b')])
})
hyp = evaluator.Outcome(label_lists={
'a':
annotations.LabelList(labels=[annotations.Label('a b d f i b')])
})
result = evaluator.ASREvaluator().do_evaluate(ref, hyp)
assert len(result.utt_to_label_pairs) == 1
assert result.utt_to_label_pairs['a'] == [
alignment.LabelPair(annotations.Label('a'),
annotations.Label('a')),
alignment.LabelPair(annotations.Label('b'),
annotations.Label('b')),
alignment.LabelPair(annotations.Label('a'), None),
alignment.LabelPair(annotations.Label('d'),
annotations.Label('d')),
alignment.LabelPair(annotations.Label('f'),
annotations.Label('f')),
alignment.LabelPair(annotations.Label('a'),
annotations.Label('i')),
alignment.LabelPair(annotations.Label('b'),
annotations.Label('b')),
]
def test_evaluate_with_multiple_labels(self):
ref = evaluator.Outcome(
label_lists={
'a':
annotations.LabelList(labels=[
annotations.Label('a b', start=0, end=3),
annotations.Label('a d', start=3, end=5),
annotations.Label('f a b', start=5, end=6)
])
})
hyp = evaluator.Outcome(label_lists={
'a':
annotations.LabelList(labels=[annotations.Label('a b d f i b')])
})
result = evaluator.ASREvaluator().do_evaluate(ref, hyp)
assert len(result.utt_to_label_pairs) == 1
assert result.utt_to_label_pairs['a'] == [
alignment.LabelPair(annotations.Label('a'),
annotations.Label('a')),
alignment.LabelPair(annotations.Label('b'),
annotations.Label('b')),
alignment.LabelPair(annotations.Label('a'), None),
alignment.LabelPair(annotations.Label('d'),
annotations.Label('d')),
alignment.LabelPair(annotations.Label('f'),
annotations.Label('f')),
alignment.LabelPair(annotations.Label('a'),
annotations.Label('i')),
alignment.LabelPair(annotations.Label('b'),
annotations.Label('b')),
]
def test_align_endless_labels(self):
ref_ll = [
annotations.Label('a', 4.2, 8.5),
annotations.Label('b', 13.1, 19.23)
]
hyp_ll = [
annotations.Label('x', 9.2, float('inf'))
]
result = alignment.FullMatchingAligner(0.1).align(ref_ll, hyp_ll)
assert sorted(result) == sorted([
alignment.LabelPair(annotations.Label('a', 4.2, 8.5),
None),
alignment.LabelPair(annotations.Label('b', 13.1, 19.23),
annotations.Label('x', 9.2, float('inf')))
])
def test_align_deletion(self):
ref_ll = [annotations.Label('a', 4.2, 8.5)]
hyp_ll = []
result = alignment.FullMatchingAligner(0.1).align(ref_ll, hyp_ll)
assert result == [
alignment.LabelPair(annotations.Label('a', 4.2, 8.5), None)
]
def test_align_insertion(self):
ref_ll = []
hyp_ll = [annotations.Label('y', 7.6, 15.2)]
result = alignment.FullMatchingAligner(0.1).align(ref_ll, hyp_ll)
assert result == [
alignment.LabelPair(None, annotations.Label('y', 7.6, 15.2))
]
def test_align(self):
ref_ll = [
annotations.Label('a', 4.2, 8.5),
annotations.Label('b', 13.1, 19.23)
]
hyp_ll = [
annotations.Label('x', 3.2, 5.4),
annotations.Label('y', 7.6, 15.2)
]
result = alignment.FullMatchingAligner(0.1).align(ref_ll, hyp_ll)
assert result == [
alignment.LabelPair(annotations.Label('a', 4.2, 8.5),
annotations.Label('x', 3.2, 5.4)),
alignment.LabelPair(annotations.Label('a', 4.2, 8.5),
annotations.Label('y', 7.6, 15.2)),
alignment.LabelPair(annotations.Label('b', 13.1, 19.23),
annotations.Label('y', 7.6, 15.2))
]
def test_align_empty_ref_returns_insertions(self):
ll_ref = []
ll_hyp = [annotations.Label('greasy', 1.4, 1.9)]
aligner = alignment.BipartiteMatchingAligner(
substitution_penalty=2,
non_overlap_penalty_weight=1
)
matches = aligner.align(ll_ref, ll_hyp)
assert matches == [
alignment.LabelPair(None, annotations.Label('greasy', 1.4, 1.9))
]
def test_evaluate_computes_correct_alignment(self, ll_ref, ll_hyp):
aligner = alignment.BipartiteMatchingAligner()
result = evaluator.EventEvaluator(aligner).evaluate(ll_ref, ll_hyp)
expected_matches = [
alignment.LabelPair(annotations.Label('up', start=5.28, end=5.99),
annotations.Label('up', start=5.20, end=5.88)),
alignment.LabelPair(annotations.Label('down', start=10.35, end=11.12),
annotations.Label('right', start=10.30, end=11.08)),
alignment.LabelPair(annotations.Label('right', start=20.87, end=22.01), None),
alignment.LabelPair(annotations.Label('up', start=33.00, end=33.4), None),
alignment.LabelPair(annotations.Label('up', start=33.4, end=33.8), None),
alignment.LabelPair(annotations.Label('down', start=39.28, end=40.0),
annotations.Label('down', start=39.27, end=40.01)),
alignment.LabelPair(None, annotations.Label('up', start=32.00, end=32.5)),
alignment.LabelPair(None, annotations.Label('up', start=34.2, end=34.8)),
alignment.LabelPair(None, annotations.Label('left', start=39.3, end=39.9))
]
assert isinstance(result, evaluator.EventEvaluation)
assert sorted(expected_matches) == sorted(result.label_pairs)
def sample_confusion():
conf = confusion.EventConfusion('up')
conf.correct_pairs.extend([
alignment.LabelPair(annotations.Label('up', start=5.28, end=5.99),
annotations.Label('up', start=5.20, end=5.88)),
alignment.LabelPair(annotations.Label('up', start=39.28, end=40.0),
annotations.Label('down', start=39.27, end=40.01))
])
conf.insertion_pairs.extend([
alignment.LabelPair(None, annotations.Label('up',
start=32.00,
end=32.5)),
alignment.LabelPair(None, annotations.Label('up', start=34.2,
end=34.8)),
alignment.LabelPair(None, annotations.Label('up', start=55.3,
end=56.9))
])
conf.deletion_pairs.extend([
alignment.LabelPair(annotations.Label('up', start=20.87, end=22.01),
None),
alignment.LabelPair(annotations.Label('up', start=33.00, end=33.4),
None),
alignment.LabelPair(annotations.Label('up', start=33.4, end=33.8),
None),
])
conf.substitution_pairs['right'].extend([
alignment.LabelPair(annotations.Label('up', start=10.35, end=11.12),
annotations.Label('right', start=10.30,
end=11.18)),
])
conf.substitution_pairs['down'].extend([
alignment.LabelPair(annotations.Label('up', start=74.28, end=75.0),
annotations.Label('down', start=74.17, end=75.01))
])
conf.substitution_out_pairs['left'].extend([
alignment.LabelPair(annotations.Label('left', start=15.35, end=16.12),
annotations.Label('up', start=15.4, end=16.18)),
])
conf.substitution_out_pairs['up'].extend([
alignment.LabelPair(annotations.Label('down', start=84.28, end=85.09),
annotations.Label('up', start=84.17, end=85.01))
])
return conf
