def evaluate(summary, sumref, debug=False):
sumstring = sumtostr(summary)
sumtuple = sumtotup(summary)
refstring = sumtostr(sumref)
reftuple = sumtotup(sumref)
summodel = TfDocumentModel(sumstring, Tokenizer("english"))
refmodel = TfDocumentModel(refstring, Tokenizer("english"))
if debug:
print(reftuple)
print(sumtuple)
cos_val = cosine_similarity(summodel, refmodel)
unit_val = unit_overlap(summodel, refmodel)
precision_val = precision(sumtuple, reftuple)
recall_val = recall(sumtuple, reftuple)
f_val = f_score(sumtuple, reftuple)
return cos_val, unit_val, precision_val, recall_val, f_val
def test_recall_equals(self):
sentences = ("s1", "s2", "s3", "s4", "s5")
result = recall(sentences, sentences)
self.assertAlmostEqual(result, 1.0)
def test_recall_evaluated_smaller(self):
result = recall(("s1", ), ("s1", "s2", "s3", "s4", "s5"))
self.assertAlmostEqual(result, 0.20)
def test_recall_reference_smaller(self):
result = recall(("s1", "s2", "s3", "s4", "s5"), ("s1", ))
self.assertAlmostEqual(result, 1.0)
def test_recall_no_match(self):
result = recall(("s1", "s2", "s3", "s4", "s5"), ("s6", "s7", "s8"))
self.assertEqual(result, 0.0)
def test_recall_no_match(self):
result = recall(("s1", "s2", "s3", "s4", "s5"), ("s6", "s7", "s8"))
self.assertEqual(result, 0.0)
def test_recall_evaluated_smaller():
result = recall(("s1", ), ("s1", "s2", "s3", "s4", "s5"))
assert result == approx(0.20)
def test_recall_no_match():
result = recall(("s1", "s2", "s3", "s4", "s5"), ("s6", "s7", "s8"))
assert result == 0.0
def test_recall_no_match():
result = recall(("s1", "s2", "s3", "s4", "s5"), ("s6", "s7", "s8"))
assert result == 0.0
def test_recall_empty_reference():
with pytest.raises(ValueError):
recall(("s1", "s2", "s3", "s4", "s5"), ())
def test_recall_empty_evaluated():
with pytest.raises(ValueError):
recall((), ("s1", "s2", "s3", "s4", "s5"))
def test_recall_equals(self):
sentences = ("s1", "s2", "s3", "s4", "s5")
result = recall(sentences, sentences)
self.assertAlmostEqual(result, 1.0)
def test_recall_evaluated_smaller(self):
result = recall(("s1",), ("s1", "s2", "s3", "s4", "s5"))
self.assertAlmostEqual(result, 0.20)
def test_recall_reference_smaller(self):
result = recall(("s1", "s2", "s3", "s4", "s5"), ("s1",))
self.assertAlmostEqual(result, 1.0)
def test_recall_empty_evaluated():
with pytest.raises(ValueError):
recall((), ("s1", "s2", "s3", "s4", "s5"))
def test_recall_empty_reference():
with pytest.raises(ValueError):
recall(("s1", "s2", "s3", "s4", "s5"), ())
def test_recall_reference_smaller():
result = recall(("s1", "s2", "s3", "s4", "s5"), ("s1",))
assert result == approx(1.0)
def test_recall_reference_smaller():
result = recall(("s1", "s2", "s3", "s4", "s5"), ("s1", ))
assert result == approx(1.0)
def test_recall_evaluated_smaller():
result = recall(("s1",), ("s1", "s2", "s3", "s4", "s5"))
assert result == approx(0.20)
def test_recall_equals():
sentences = ("s1", "s2", "s3", "s4", "s5")
result = recall(sentences, sentences)
assert result == approx(1.0)
def test_recall_equals():
sentences = ("s1", "s2", "s3", "s4", "s5")
result = recall(sentences, sentences)
assert result == approx(1.0)