def testEvaluateRankingRandomZero(self):
     test_ndcgs = {
         "1.0000": 0,
         "0.9769": 0,
         "0.9688": 0,
         "0.6540": 0,
         "0.6227": 0,
         "0.5312": 0,
         "0.5082": 0
     }
     trials = 0
     MAX_TRIALS = 1000
     ev = LetorNdcgEval()
     dot_prod = np.dot(self.query.get_feature_vectors(),
                       self.zero_weights.transpose())
     while trials < MAX_TRIALS and 0 in test_ndcgs.values():
         trials += 1
         ranking = ev._sort_docids_by_score(self.query.get_docids(),
                                            dot_prod,
                                            ties="random")
         observed_ndcg = "%.4f" % ev.evaluate_ranking(ranking, self.query)
         if observed_ndcg in test_ndcgs:
             test_ndcgs[observed_ndcg] += 1
         else:
             print "unknown ndcg: ", observed_ndcg
     print "Observed all test ndcgs within %d trials." % trials
     for ndcg, count in test_ndcgs.items():
         self.assertNotEqual(0, count, "Test failed for %s" % ndcg)
Beispiel #2
0
 def testEvaluateRankingFirstZero(self):
     ev = LetorNdcgEval()
     dot_prod = np.dot(self.query.get_feature_vectors(),
         self.zero_weights.transpose())
     ranking = ev._sort_docids_by_score(self.query.get_docids(),
         dot_prod, ties="first")
     self.assertEquals(1, ev.evaluate_ranking(ranking, self.query))
 def testEvaluateRankingFirstZero(self):
     ev = LetorNdcgEval()
     dot_prod = np.dot(self.query.get_feature_vectors(),
                       self.zero_weights.transpose())
     ranking = ev._sort_docids_by_score(self.query.get_docids(),
                                        dot_prod,
                                        ties="first")
     self.assertEquals(1, ev.evaluate_ranking(ranking, self.query))
Beispiel #4
0
 def testEvaluateRankingLastZeroWithCutoffLetor(self):
     ev = LetorNdcgEval()
     dot_prod = np.dot(self.query.get_feature_vectors(),
         self.zero_weights.transpose())
     ranking = ev._sort_docids_by_score(self.query.get_docids(),
         dot_prod, ties="last")
     self.assertAlmostEquals(0.5081831, ev.evaluate_ranking(ranking,
         self.query, cutoff=10))
Beispiel #5
0
 def testEvaluateRankingLast(self):
     ev = LetorNdcgEval()
     dot_prod = np.dot(self.query.get_feature_vectors(),
         self.weights.transpose())
     ranking = ev._sort_docids_by_score(self.query.get_docids(),
         dot_prod, ties="last")
     self.assertAlmostEquals(0.53125, ev.evaluate_ranking(ranking,
         self.query), msg="ranking " + ",".join(str(d) for d in ranking))
 def testEvaluateRankingLastZeroWithCutoffLetor(self):
     ev = LetorNdcgEval()
     dot_prod = np.dot(self.query.get_feature_vectors(),
                       self.zero_weights.transpose())
     ranking = ev._sort_docids_by_score(self.query.get_docids(),
                                        dot_prod,
                                        ties="last")
     self.assertAlmostEquals(
         0.5081831, ev.evaluate_ranking(ranking, self.query, cutoff=10))
 def testEvaluateRankingFirst(self):
     ev = LetorNdcgEval()
     dot_prod = np.dot(self.query.get_feature_vectors(),
                       self.weights.transpose())
     ranking = ev._sort_docids_by_score(self.query.get_docids(),
                                        dot_prod,
                                        ties="first")
     self.assertEquals(0.53125,
                       ev.evaluate_ranking(ranking, self.query),
                       msg="ranking " + ",".join(str(d) for d in ranking))
Beispiel #8
0
 def testEvaluateRankingRandomZero(self):
     test_ndcgs = {"1.0000": 0, "0.9769": 0, "0.9688": 0, "0.6540": 0,
         "0.6227": 0, "0.5312": 0, "0.5082": 0}
     trials = 0
     MAX_TRIALS = 1000
     ev = LetorNdcgEval()
     dot_prod = np.dot(self.query.get_feature_vectors(),
         self.zero_weights.transpose())
     while trials < MAX_TRIALS and 0 in test_ndcgs.values():
         trials += 1
         ranking = ev._sort_docids_by_score(self.query.get_docids(),
             dot_prod, ties="random")
         observed_ndcg = "%.4f" % ev.evaluate_ranking(ranking, self.query)
         if observed_ndcg in test_ndcgs:
             test_ndcgs[observed_ndcg] += 1
         else:
             print "unknown ndcg: ", observed_ndcg
     print "Observed all test ndcgs within %d trials." % trials
     for ndcg, count in test_ndcgs.items():
         self.assertNotEqual(0, count, "Test failed for %s" % ndcg)
 def testGetLetorNdcg(self):
     ev = LetorNdcgEval()
     self.assertAlmostEqual(ev.get_ndcg([0, 0, 1, 0], [1, 0, 0, 0]),
                            0.6309298)
     self.assertEqual(ev.get_ndcg([0, 1, 0, 0], [1, 0, 0, 0]), 1)
     self.assertEqual(ev.get_ndcg([0, 0, 0, 0], [1, 0, 0, 0]), 0)
 def testEvaluateOneLastZero(self):
     ev = LetorNdcgEval()
     self.assertAlmostEquals(
         0.5081831,
         ev.evaluate_one(self.zero_weights, self.query, ties="last"))
 def testEvaluateOneFirstZero(self):
     ev = LetorNdcgEval()
     self.assertEquals(
         1, ev.evaluate_one(self.zero_weights, self.query, ties="first"))
Beispiel #12
0
 def testGetLetorNdcg(self):
     ev = LetorNdcgEval()
     self.assertAlmostEqual(ev.get_ndcg([0, 0, 1, 0], [1, 0, 0, 0]),
         0.6309298)
     self.assertEqual(ev.get_ndcg([0, 1, 0, 0], [1, 0, 0, 0]), 1)
     self.assertEqual(ev.get_ndcg([0, 0, 0, 0], [1, 0, 0, 0]), 0)
Beispiel #13
0
 def testEvaluateOneLastZero(self):
     ev = LetorNdcgEval()
     self.assertAlmostEquals(0.5081831, ev.evaluate_one(self.zero_weights,
         self.query, ties="last"))
Beispiel #14
0
 def testEvaluateOneFirstZero(self):
     ev = LetorNdcgEval()
     self.assertEquals(1, ev.evaluate_one(self.zero_weights, self.query,
         ties="first"))