def test_evaluate(self):
ref = evaluator.Outcome(
label_lists={
'a':
annotations.LabelList(
labels=[annotations.Label('a b a d f a b')])
})
hyp = evaluator.Outcome(label_lists={
'a':
annotations.LabelList(labels=[annotations.Label('a b d f i b')])
})
result = evaluator.ASREvaluator().do_evaluate(ref, hyp)
assert len(result.utt_to_label_pairs) == 1
assert result.utt_to_label_pairs['a'] == [
alignment.LabelPair(annotations.Label('a'),
annotations.Label('a')),
alignment.LabelPair(annotations.Label('b'),
annotations.Label('b')),
alignment.LabelPair(annotations.Label('a'), None),
alignment.LabelPair(annotations.Label('d'),
annotations.Label('d')),
alignment.LabelPair(annotations.Label('f'),
annotations.Label('f')),
alignment.LabelPair(annotations.Label('a'),
annotations.Label('i')),
alignment.LabelPair(annotations.Label('b'),
annotations.Label('b')),
]
def test_evaluate_with_multiple_labels(self):
ref = evaluator.Outcome(
label_lists={
'a':
annotations.LabelList(labels=[
annotations.Label('a b', start=0, end=3),
annotations.Label('a d', start=3, end=5),
annotations.Label('f a b', start=5, end=6)
])
})
hyp = evaluator.Outcome(label_lists={
'a':
annotations.LabelList(labels=[annotations.Label('a b d f i b')])
})
result = evaluator.ASREvaluator().do_evaluate(ref, hyp)
assert len(result.utt_to_label_pairs) == 1
assert result.utt_to_label_pairs['a'] == [
alignment.LabelPair(annotations.Label('a'),
annotations.Label('a')),
alignment.LabelPair(annotations.Label('b'),
annotations.Label('b')),
alignment.LabelPair(annotations.Label('a'), None),
alignment.LabelPair(annotations.Label('d'),
annotations.Label('d')),
alignment.LabelPair(annotations.Label('f'),
annotations.Label('f')),
alignment.LabelPair(annotations.Label('a'),
annotations.Label('i')),
alignment.LabelPair(annotations.Label('b'),
annotations.Label('b')),
]
def evaluate(labels, preds):
ref = {
k: annotations.LabelList.create_single(v)
for k, v in labels.items()
}
hyp = {k: annotations.LabelList.create_single(v) for k, v in preds.items()}
ref_out = evaluator.Outcome(ref)
hyp_out = evaluator.Outcome(hyp)
result = evalmate.ASREvaluator().evaluate(ref_out, hyp_out)
return result
def test_evaluate_with_empty_hyp(self):
ref = evaluator.Outcome(
label_lists={
'a':
annotations.LabelList(labels=[
annotations.Label('one', 2.5, 4.5),
annotations.Label('two', 10.5, 11.5),
]),
'b':
annotations.LabelList(labels=[
annotations.Label('one', 1.5, 1.9),
]),
'c':
annotations.LabelList(labels=[
annotations.Label('two', 4.5, 4.9),
annotations.Label('two', 10.5, 11.5),
]),
})
hyp = evaluator.Outcome(
label_lists={
'a':
annotations.LabelList(labels=[
annotations.Label('one', 2.5, 4.5),
annotations.Label('two', 10.5, 11.5),
]),
'b':
annotations.LabelList(labels=[]),
'c':
annotations.LabelList(labels=[
annotations.Label('two', 4.5, 4.9),
annotations.Label('two', 10.5, 11.5),
]),
})
result = evaluator.KWSEvaluator().evaluate(ref, hyp)
assert result.confusion.total == 5