def test_different_mlqa(self):
ref = "this is a string"
self.assertDictClose(
metrics.mlqa([[ref, ref], [ref, ref]], ["", ""], lang="en"), {
"em": 0,
"f1": 0
})
def test_article_drop_mlqa(self):
ref = "this unas a string"
pred = "this a string"
self.assertDictClose(metrics.mlqa([[ref]], [pred], lang="es"), {
"em": 100,
"f1": 100,
})
def test_same_mlqa(self):
ref = "this is a string"
self.assertDictClose(
metrics.mlqa([["", ref], [ref, ref]], [ref, ref], lang="en"), {
"em": 100,
"f1": 100,
})
def test_mlqa_small(self):
self.assertDictClose(
metrics.mlqa([["abc abd", "$$$$"]], ["abd"], lang="en"),
{
"f1": 100 * 2.0 / 3.0,
"em": 0.
},
)