def testProbabilisticInterleaveWithDeterministicRankers(self): pi = ProbabilisticInterleave(None) # test a few possible interleavings r1 = DeterministicRankingFunction(None, self.weights_1) r2 = DeterministicRankingFunction(None, self.weights_2) test_lists = {"0,1,3,2": 0, "1,0,3,2": 0, "1,3,0,2": 0, "1,3,2,0": 0} trials = 0 MAX_TRIALS = 10000 while trials < MAX_TRIALS and 0 in test_lists.values(): trials += 1 (l, a) = pi.interleave(r1, r2, self.query, 10) list_str = ",".join(str(a) for a in l.tolist()) self.assertIn(list_str, test_lists.keys()) test_lists[list_str] += 1 for list_str, count in test_lists.items(): self.assertNotEqual(0, count, "Interleave failed for: %s" % list_str) # test interleaving outcomes context = (None, r1, r2) self.assertEqual(pi.infer_outcome([0, 1, 2, 3], context, [0, 0, 0, 0], self.query), 0, "No clicks, outcome should be 0.") self.assertEqual(pi.infer_outcome([0, 1, 2, 3], context, [1, 0, 0, 0], self.query), 0, "No possible assignment, outcome should be 0.") o = pi.infer_outcome([1, 0, 3, 2], context, [1, 0, 0, 0], self.query) self.assertAlmostEquals(o, -0.0625, 4, "Ranker 1 should win (o = %.4f)." % o) o = pi.infer_outcome([0, 1, 3, 2], context, [1, 0, 0, 0], self.query) self.assertAlmostEquals(o, 0.0625, 4, "Ranker 2 should win (o = %.4f)." % o) # test get_probability_of_list p = pi.get_probability_of_list([1, 0, 3, 2], context, self.query) self.assertEqual(p, 0.25, "Probability of the most " "likely list. p = %g" % p)
def testProbabilisticInterleaveWithDeterministicRankers(self): pi = ProbabilisticInterleave(None) # test a few possible interleavings r1 = DeterministicRankingFunction(None, self.weights_1) r2 = DeterministicRankingFunction(None, self.weights_2) test_lists = {"0,1,3,2": 0, "1,0,3,2": 0, "1,3,0,2": 0, "1,3,2,0": 0} trials = 0 MAX_TRIALS = 10000 while trials < MAX_TRIALS and 0 in test_lists.values(): trials += 1 (l, a) = pi.interleave(r1, r2, self.query, 10) list_str = ",".join(str(a) for a in l.tolist()) self.assertIn(list_str, test_lists.keys()) test_lists[list_str] += 1 for list_str, count in test_lists.items(): self.assertNotEqual(0, count, "Interleave failed for: %s" % list_str) # test interleaving outcomes context = (None, r1, r2) self.assertEqual( pi.infer_outcome([0, 1, 2, 3], context, [0, 0, 0, 0], self.query), 0, "No clicks, outcome should be 0.") self.assertEqual( pi.infer_outcome([0, 1, 2, 3], context, [1, 0, 0, 0], self.query), 0, "No possible assignment, outcome should be 0.") o = pi.infer_outcome([1, 0, 3, 2], context, [1, 0, 0, 0], self.query) self.assertAlmostEquals(o, -0.0625, 4, "Ranker 1 should win (o = %.4f)." % o) o = pi.infer_outcome([0, 1, 3, 2], context, [1, 0, 0, 0], self.query) self.assertAlmostEquals(o, 0.0625, 4, "Ranker 2 should win (o = %.4f)." % o) # test get_probability_of_list p = pi.get_probability_of_list([1, 0, 3, 2], context, self.query) self.assertEqual(p, 0.25, "Probability of the most " "likely list. p = %g" % p)
def testProbabilisticInterleave(self): pi = ProbabilisticInterleave(None) r1 = ProbabilisticRankingFunction(3, self.weights_1) r2 = ProbabilisticRankingFunction(3, self.weights_2) context = (None, r1, r2) # test get_probability_of_list p = pi.get_probability_of_list([1, 0, 3, 2], context, self.query) self.assertAlmostEquals(p, 0.182775, 6, "Probability of the most " "likely list. p = %.6f" % p) # test a few possible interleavings test_lists = {"0,1,2,3": 0, "0,1,3,2": 0, "0,2,1,3": 0, "0,2,3,1": 0, "0,3,1,2": 0, "0,3,2,1": 0, "1,0,2,3": 0, "1,0,3,2": 0, "1,2,0,3": 0, "1,2,3,0": 0, "1,3,0,2": 0, "1,3,2,0": 0, "2,0,1,3": 0, "2,0,3,1": 0, "2,1,0,3": 0, "2,1,3,0": 0, "2,3,0,1": 0, "2,3,1,0": 0, "3,0,1,2": 0, "3,0,2,1": 0, "3,1,0,2": 0, "3,1,2,0": 0, "3,2,0,1": 0, "3,2,1,0": 0} trials = 0 MAX_TRIALS = 100000 while trials < MAX_TRIALS and 0 in test_lists.values(): trials += 1 (l, _) = pi.interleave(r1, r2, self.query, 10) list_str = ",".join(str(a) for a in l.tolist()) self.assertIn(list_str, test_lists.keys()) test_lists[list_str] += 1 for list_str, count in test_lists.items(): self.assertNotEqual(0, count, "Interleave failed for: %s" % list_str) # test interleaving outcomes self.assertEqual(pi.infer_outcome([0, 1, 2, 3], context, [0, 0, 0, 0], self.query), 0, "No clicks, outcome should be 0.") o = pi.infer_outcome([1, 0, 3, 2], context, [1, 0, 0, 0], self.query) self.assertAlmostEquals(o, -0.0486, 4, "Ranker 1 should win (o = %.4f)." % o) o = pi.infer_outcome([0, 1, 3, 2], context, [1, 0, 0, 0], self.query) self.assertAlmostEquals(o, 0.0606, 4, "Ranker 2 should win (o = %.4f)." % o) # from the example in CIKM 2011 weight_str_1 = "0 0 1 0 -1 0" weights_1 = np.asarray([float(x) for x in weight_str_1.split()]) weight_str_2 = "1 0 0 0 -1 0" weights_2 = np.asarray([float(x) for x in weight_str_2.split()]) r1 = ProbabilisticRankingFunction(3, weights_1) r2 = ProbabilisticRankingFunction(3, weights_2) context = (None, r2, r1) o = pi.infer_outcome([0, 1, 2, 3], context, [0, 1, 1, 0], self.query) self.assertAlmostEquals(o, 0.0046, 4, "Ranker 2 should win again (o = %.4f)." % o) # click on one before last document o = pi.infer_outcome([3, 1, 0, 2], context, [0, 0, 1, 0], self.query) self.assertAlmostEquals(o, -0.0496, 4, "Ranker 1 should win with click on doc 0 (o = %.4f)." % o) # click on last document o = pi.infer_outcome([3, 1, 2, 0], context, [0, 0, 0, 1], self.query) self.assertAlmostEquals(o, 0.0, 4, "Tie for click on last doc (o = %.4f)." % o)
def testProbabilisticInterleave(self): pi = ProbabilisticInterleave(None) r1 = ProbabilisticRankingFunction(3, self.weights_1) r2 = ProbabilisticRankingFunction(3, self.weights_2) context = (None, r1, r2) # test get_probability_of_list p = pi.get_probability_of_list([1, 0, 3, 2], context, self.query) self.assertAlmostEquals( p, 0.182775, 6, "Probability of the most " "likely list. p = %.6f" % p) # test a few possible interleavings test_lists = { "0,1,2,3": 0, "0,1,3,2": 0, "0,2,1,3": 0, "0,2,3,1": 0, "0,3,1,2": 0, "0,3,2,1": 0, "1,0,2,3": 0, "1,0,3,2": 0, "1,2,0,3": 0, "1,2,3,0": 0, "1,3,0,2": 0, "1,3,2,0": 0, "2,0,1,3": 0, "2,0,3,1": 0, "2,1,0,3": 0, "2,1,3,0": 0, "2,3,0,1": 0, "2,3,1,0": 0, "3,0,1,2": 0, "3,0,2,1": 0, "3,1,0,2": 0, "3,1,2,0": 0, "3,2,0,1": 0, "3,2,1,0": 0 } trials = 0 MAX_TRIALS = 100000 while trials < MAX_TRIALS and 0 in test_lists.values(): trials += 1 (l, _) = pi.interleave(r1, r2, self.query, 10) list_str = ",".join(str(a) for a in l.tolist()) self.assertIn(list_str, test_lists.keys()) test_lists[list_str] += 1 for list_str, count in test_lists.items(): self.assertNotEqual(0, count, "Interleave failed for: %s" % list_str) # test interleaving outcomes self.assertEqual( pi.infer_outcome([0, 1, 2, 3], context, [0, 0, 0, 0], self.query), 0, "No clicks, outcome should be 0.") o = pi.infer_outcome([1, 0, 3, 2], context, [1, 0, 0, 0], self.query) self.assertAlmostEquals(o, -0.0486, 4, "Ranker 1 should win (o = %.4f)." % o) o = pi.infer_outcome([0, 1, 3, 2], context, [1, 0, 0, 0], self.query) self.assertAlmostEquals(o, 0.0606, 4, "Ranker 2 should win (o = %.4f)." % o) # from the example in CIKM 2011 weight_str_1 = "0 0 1 0 -1 0" weights_1 = np.asarray([float(x) for x in weight_str_1.split()]) weight_str_2 = "1 0 0 0 -1 0" weights_2 = np.asarray([float(x) for x in weight_str_2.split()]) r1 = ProbabilisticRankingFunction(3, weights_1) r2 = ProbabilisticRankingFunction(3, weights_2) context = (None, r2, r1) o = pi.infer_outcome([0, 1, 2, 3], context, [0, 1, 1, 0], self.query) self.assertAlmostEquals(o, 0.0046, 4, "Ranker 2 should win again (o = %.4f)." % o) # click on one before last document o = pi.infer_outcome([3, 1, 0, 2], context, [0, 0, 1, 0], self.query) self.assertAlmostEquals( o, -0.0496, 4, "Ranker 1 should win with click on doc 0 (o = %.4f)." % o) # click on last document o = pi.infer_outcome([3, 1, 2, 0], context, [0, 0, 0, 1], self.query) self.assertAlmostEquals(o, 0.0, 4, "Tie for click on last doc (o = %.4f)." % o)