def testEvaluateRankingRandomZero(self):
test_ndcgs = {
"1.0000": 0,
"0.9769": 0,
"0.9688": 0,
"0.6540": 0,
"0.6227": 0,
"0.5312": 0,
"0.5082": 0
}
trials = 0
MAX_TRIALS = 1000
ev = LetorNdcgEval()
dot_prod = np.dot(self.query.get_feature_vectors(),
self.zero_weights.transpose())
while trials < MAX_TRIALS and 0 in test_ndcgs.values():
trials += 1
ranking = ev._sort_docids_by_score(self.query.get_docids(),
dot_prod,
ties="random")
observed_ndcg = "%.4f" % ev.evaluate_ranking(ranking, self.query)
if observed_ndcg in test_ndcgs:
test_ndcgs[observed_ndcg] += 1
else:
print "unknown ndcg: ", observed_ndcg
print "Observed all test ndcgs within %d trials." % trials
for ndcg, count in test_ndcgs.items():
self.assertNotEqual(0, count, "Test failed for %s" % ndcg)
def testEvaluateRankingFirstZero(self):
ev = LetorNdcgEval()
dot_prod = np.dot(self.query.get_feature_vectors(),
self.zero_weights.transpose())
ranking = ev._sort_docids_by_score(self.query.get_docids(),
dot_prod, ties="first")
self.assertEquals(1, ev.evaluate_ranking(ranking, self.query))
def testEvaluateRankingLastZeroWithCutoffLetor(self):
ev = LetorNdcgEval()
dot_prod = np.dot(self.query.get_feature_vectors(),
self.zero_weights.transpose())
ranking = ev._sort_docids_by_score(self.query.get_docids(),
dot_prod, ties="last")
self.assertAlmostEquals(0.5081831, ev.evaluate_ranking(ranking,
self.query, cutoff=10))
def testEvaluateRankingLast(self):
ev = LetorNdcgEval()
dot_prod = np.dot(self.query.get_feature_vectors(),
self.weights.transpose())
ranking = ev._sort_docids_by_score(self.query.get_docids(),
dot_prod, ties="last")
self.assertAlmostEquals(0.53125, ev.evaluate_ranking(ranking,
self.query), msg="ranking " + ",".join(str(d) for d in ranking))
def testEvaluateRankingFirstZero(self):
ev = LetorNdcgEval()
dot_prod = np.dot(self.query.get_feature_vectors(),
self.zero_weights.transpose())
ranking = ev._sort_docids_by_score(self.query.get_docids(),
dot_prod,
ties="first")
self.assertEquals(1, ev.evaluate_ranking(ranking, self.query))
def testEvaluateRankingLastZeroWithCutoffLetor(self):
ev = LetorNdcgEval()
dot_prod = np.dot(self.query.get_feature_vectors(),
self.zero_weights.transpose())
ranking = ev._sort_docids_by_score(self.query.get_docids(),
dot_prod,
ties="last")
self.assertAlmostEquals(
0.5081831, ev.evaluate_ranking(ranking, self.query, cutoff=10))
def testEvaluateRankingFirst(self):
ev = LetorNdcgEval()
dot_prod = np.dot(self.query.get_feature_vectors(),
self.weights.transpose())
ranking = ev._sort_docids_by_score(self.query.get_docids(),
dot_prod,
ties="first")
self.assertEquals(0.53125,
ev.evaluate_ranking(ranking, self.query),
msg="ranking " + ",".join(str(d) for d in ranking))
def testEvaluateRankingRandomZero(self):
test_ndcgs = {"1.0000": 0, "0.9769": 0, "0.9688": 0, "0.6540": 0,
"0.6227": 0, "0.5312": 0, "0.5082": 0}
trials = 0
MAX_TRIALS = 1000
ev = LetorNdcgEval()
dot_prod = np.dot(self.query.get_feature_vectors(),
self.zero_weights.transpose())
while trials < MAX_TRIALS and 0 in test_ndcgs.values():
trials += 1
ranking = ev._sort_docids_by_score(self.query.get_docids(),
dot_prod, ties="random")
observed_ndcg = "%.4f" % ev.evaluate_ranking(ranking, self.query)
if observed_ndcg in test_ndcgs:
test_ndcgs[observed_ndcg] += 1
else:
print "unknown ndcg: ", observed_ndcg
print "Observed all test ndcgs within %d trials." % trials
for ndcg, count in test_ndcgs.items():
self.assertNotEqual(0, count, "Test failed for %s" % ndcg)
def testGetLetorNdcg(self):
ev = LetorNdcgEval()
self.assertAlmostEqual(ev.get_ndcg([0, 0, 1, 0], [1, 0, 0, 0]),
0.6309298)
self.assertEqual(ev.get_ndcg([0, 1, 0, 0], [1, 0, 0, 0]), 1)
self.assertEqual(ev.get_ndcg([0, 0, 0, 0], [1, 0, 0, 0]), 0)
def testEvaluateOneLastZero(self):
ev = LetorNdcgEval()
self.assertAlmostEquals(0.5081831, ev.evaluate_one(self.zero_weights,
self.query, ties="last"))
def testEvaluateOneFirstZero(self):
ev = LetorNdcgEval()
self.assertEquals(1, ev.evaluate_one(self.zero_weights, self.query,
ties="first"))
def testGetLetorNdcg(self):
ev = LetorNdcgEval()
self.assertAlmostEqual(ev.get_ndcg([0, 0, 1, 0], [1, 0, 0, 0]),
0.6309298)
self.assertEqual(ev.get_ndcg([0, 1, 0, 0], [1, 0, 0, 0]), 1)
self.assertEqual(ev.get_ndcg([0, 0, 0, 0], [1, 0, 0, 0]), 0)
def testEvaluateOneLastZero(self):
ev = LetorNdcgEval()
self.assertAlmostEquals(
0.5081831,
ev.evaluate_one(self.zero_weights, self.query, ties="last"))
def testEvaluateOneFirstZero(self):
ev = LetorNdcgEval()
self.assertEquals(
1, ev.evaluate_one(self.zero_weights, self.query, ties="first"))