def testProbabilisticInterleaveWithDeterministicRankers(self):
pi = ProbabilisticInterleave(None)
# test a few possible interleavings
r1 = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
test_lists = {"0,1,3,2": 0, "1,0,3,2": 0, "1,3,0,2": 0, "1,3,2,0": 0}
trials = 0
MAX_TRIALS = 10000
while trials < MAX_TRIALS and 0 in test_lists.values():
trials += 1
(l, a) = pi.interleave(r1, r2, self.query, 10)
list_str = ",".join(str(a) for a in l.tolist())
self.assertIn(list_str, test_lists.keys())
test_lists[list_str] += 1
for list_str, count in test_lists.items():
self.assertNotEqual(0, count,
"Interleave failed for: %s" % list_str)
# test interleaving outcomes
context = (None, r1, r2)
self.assertEqual(pi.infer_outcome([0, 1, 2, 3], context, [0, 0, 0, 0],
self.query), 0, "No clicks, outcome should be 0.")
self.assertEqual(pi.infer_outcome([0, 1, 2, 3], context, [1, 0, 0, 0],
self.query), 0, "No possible assignment, outcome should be 0.")
o = pi.infer_outcome([1, 0, 3, 2], context, [1, 0, 0, 0], self.query)
self.assertAlmostEquals(o, -0.0625, 4,
"Ranker 1 should win (o = %.4f)." % o)
o = pi.infer_outcome([0, 1, 3, 2], context, [1, 0, 0, 0], self.query)
self.assertAlmostEquals(o, 0.0625, 4,
"Ranker 2 should win (o = %.4f)." % o)
# test get_probability_of_list
p = pi.get_probability_of_list([1, 0, 3, 2], context, self.query)
self.assertEqual(p, 0.25, "Probability of the most "
"likely list. p = %g" % p)
def testProbabilisticInterleaveWithDeterministicRankers(self):
pi = ProbabilisticInterleave(None)
# test a few possible interleavings
r1 = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
test_lists = {"0,1,3,2": 0, "1,0,3,2": 0, "1,3,0,2": 0, "1,3,2,0": 0}
trials = 0
MAX_TRIALS = 10000
while trials < MAX_TRIALS and 0 in test_lists.values():
trials += 1
(l, a) = pi.interleave(r1, r2, self.query, 10)
list_str = ",".join(str(a) for a in l.tolist())
self.assertIn(list_str, test_lists.keys())
test_lists[list_str] += 1
for list_str, count in test_lists.items():
self.assertNotEqual(0, count,
"Interleave failed for: %s" % list_str)
# test interleaving outcomes
context = (None, r1, r2)
self.assertEqual(
pi.infer_outcome([0, 1, 2, 3], context, [0, 0, 0, 0], self.query),
0, "No clicks, outcome should be 0.")
self.assertEqual(
pi.infer_outcome([0, 1, 2, 3], context, [1, 0, 0, 0], self.query),
0, "No possible assignment, outcome should be 0.")
o = pi.infer_outcome([1, 0, 3, 2], context, [1, 0, 0, 0], self.query)
self.assertAlmostEquals(o, -0.0625, 4,
"Ranker 1 should win (o = %.4f)." % o)
o = pi.infer_outcome([0, 1, 3, 2], context, [1, 0, 0, 0], self.query)
self.assertAlmostEquals(o, 0.0625, 4,
"Ranker 2 should win (o = %.4f)." % o)
# test get_probability_of_list
p = pi.get_probability_of_list([1, 0, 3, 2], context, self.query)
self.assertEqual(p, 0.25, "Probability of the most "
"likely list. p = %g" % p)
def testProbabilisticInterleave(self):
pi = ProbabilisticInterleave(None)
r1 = ProbabilisticRankingFunction(3, self.weights_1)
r2 = ProbabilisticRankingFunction(3, self.weights_2)
context = (None, r1, r2)
# test get_probability_of_list
p = pi.get_probability_of_list([1, 0, 3, 2], context, self.query)
self.assertAlmostEquals(p, 0.182775, 6, "Probability of the most "
"likely list. p = %.6f" % p)
# test a few possible interleavings
test_lists = {"0,1,2,3": 0, "0,1,3,2": 0, "0,2,1,3": 0, "0,2,3,1": 0,
"0,3,1,2": 0, "0,3,2,1": 0, "1,0,2,3": 0, "1,0,3,2": 0,
"1,2,0,3": 0, "1,2,3,0": 0, "1,3,0,2": 0, "1,3,2,0": 0,
"2,0,1,3": 0, "2,0,3,1": 0, "2,1,0,3": 0, "2,1,3,0": 0,
"2,3,0,1": 0, "2,3,1,0": 0, "3,0,1,2": 0, "3,0,2,1": 0,
"3,1,0,2": 0, "3,1,2,0": 0, "3,2,0,1": 0, "3,2,1,0": 0}
trials = 0
MAX_TRIALS = 100000
while trials < MAX_TRIALS and 0 in test_lists.values():
trials += 1
(l, _) = pi.interleave(r1, r2, self.query, 10)
list_str = ",".join(str(a) for a in l.tolist())
self.assertIn(list_str, test_lists.keys())
test_lists[list_str] += 1
for list_str, count in test_lists.items():
self.assertNotEqual(0, count,
"Interleave failed for: %s" % list_str)
# test interleaving outcomes
self.assertEqual(pi.infer_outcome([0, 1, 2, 3], context, [0, 0, 0, 0],
self.query), 0, "No clicks, outcome should be 0.")
o = pi.infer_outcome([1, 0, 3, 2], context, [1, 0, 0, 0], self.query)
self.assertAlmostEquals(o, -0.0486, 4,
"Ranker 1 should win (o = %.4f)." % o)
o = pi.infer_outcome([0, 1, 3, 2], context, [1, 0, 0, 0], self.query)
self.assertAlmostEquals(o, 0.0606, 4,
"Ranker 2 should win (o = %.4f)." % o)
# from the example in CIKM 2011
weight_str_1 = "0 0 1 0 -1 0"
weights_1 = np.asarray([float(x) for x in weight_str_1.split()])
weight_str_2 = "1 0 0 0 -1 0"
weights_2 = np.asarray([float(x) for x in weight_str_2.split()])
r1 = ProbabilisticRankingFunction(3, weights_1)
r2 = ProbabilisticRankingFunction(3, weights_2)
context = (None, r2, r1)
o = pi.infer_outcome([0, 1, 2, 3], context, [0, 1, 1, 0], self.query)
self.assertAlmostEquals(o, 0.0046, 4,
"Ranker 2 should win again (o = %.4f)." % o)
# click on one before last document
o = pi.infer_outcome([3, 1, 0, 2], context, [0, 0, 1, 0], self.query)
self.assertAlmostEquals(o, -0.0496, 4,
"Ranker 1 should win with click on doc 0 (o = %.4f)." % o)
# click on last document
o = pi.infer_outcome([3, 1, 2, 0], context, [0, 0, 0, 1], self.query)
self.assertAlmostEquals(o, 0.0, 4,
"Tie for click on last doc (o = %.4f)." % o)
def testProbabilisticInterleave(self):
pi = ProbabilisticInterleave(None)
r1 = ProbabilisticRankingFunction(3, self.weights_1)
r2 = ProbabilisticRankingFunction(3, self.weights_2)
context = (None, r1, r2)
# test get_probability_of_list
p = pi.get_probability_of_list([1, 0, 3, 2], context, self.query)
self.assertAlmostEquals(
p, 0.182775, 6, "Probability of the most "
"likely list. p = %.6f" % p)
# test a few possible interleavings
test_lists = {
"0,1,2,3": 0,
"0,1,3,2": 0,
"0,2,1,3": 0,
"0,2,3,1": 0,
"0,3,1,2": 0,
"0,3,2,1": 0,
"1,0,2,3": 0,
"1,0,3,2": 0,
"1,2,0,3": 0,
"1,2,3,0": 0,
"1,3,0,2": 0,
"1,3,2,0": 0,
"2,0,1,3": 0,
"2,0,3,1": 0,
"2,1,0,3": 0,
"2,1,3,0": 0,
"2,3,0,1": 0,
"2,3,1,0": 0,
"3,0,1,2": 0,
"3,0,2,1": 0,
"3,1,0,2": 0,
"3,1,2,0": 0,
"3,2,0,1": 0,
"3,2,1,0": 0
}
trials = 0
MAX_TRIALS = 100000
while trials < MAX_TRIALS and 0 in test_lists.values():
trials += 1
(l, _) = pi.interleave(r1, r2, self.query, 10)
list_str = ",".join(str(a) for a in l.tolist())
self.assertIn(list_str, test_lists.keys())
test_lists[list_str] += 1
for list_str, count in test_lists.items():
self.assertNotEqual(0, count,
"Interleave failed for: %s" % list_str)
# test interleaving outcomes
self.assertEqual(
pi.infer_outcome([0, 1, 2, 3], context, [0, 0, 0, 0], self.query),
0, "No clicks, outcome should be 0.")
o = pi.infer_outcome([1, 0, 3, 2], context, [1, 0, 0, 0], self.query)
self.assertAlmostEquals(o, -0.0486, 4,
"Ranker 1 should win (o = %.4f)." % o)
o = pi.infer_outcome([0, 1, 3, 2], context, [1, 0, 0, 0], self.query)
self.assertAlmostEquals(o, 0.0606, 4,
"Ranker 2 should win (o = %.4f)." % o)
# from the example in CIKM 2011
weight_str_1 = "0 0 1 0 -1 0"
weights_1 = np.asarray([float(x) for x in weight_str_1.split()])
weight_str_2 = "1 0 0 0 -1 0"
weights_2 = np.asarray([float(x) for x in weight_str_2.split()])
r1 = ProbabilisticRankingFunction(3, weights_1)
r2 = ProbabilisticRankingFunction(3, weights_2)
context = (None, r2, r1)
o = pi.infer_outcome([0, 1, 2, 3], context, [0, 1, 1, 0], self.query)
self.assertAlmostEquals(o, 0.0046, 4,
"Ranker 2 should win again (o = %.4f)." % o)
# click on one before last document
o = pi.infer_outcome([3, 1, 0, 2], context, [0, 0, 1, 0], self.query)
self.assertAlmostEquals(
o, -0.0496, 4,
"Ranker 1 should win with click on doc 0 (o = %.4f)." % o)
# click on last document
o = pi.infer_outcome([3, 1, 2, 0], context, [0, 0, 0, 1], self.query)
self.assertAlmostEquals(o, 0.0, 4,
"Tie for click on last doc (o = %.4f)." % o)
