def testHistProbabilisticInterleaveWithoutMarginalization(self):
r1 = ProbabilisticRankingFunction(3, self.weights_1)
r2 = ProbabilisticRankingFunction(3, self.weights_2)
hpiIs = HistProbabilisticInterleave("--biased False "
"--marginalize False")
# test get_probability_of_list_and_assignment
p = hpiIs._get_probability_of_list_and_assignment([1, 3, 2, 0],
[0, 0, 0, 0], r1, r2,
self.query)
self.assertAlmostEqual(p, 0.026261, 6, "Most likely list for ranker 1."
" p = %e" % p)
# test overall outcomes
a = ([0, 1, 1, 0], r1, r2)
self.assertEqual(
hpiIs.infer_outcome([0, 1, 2, 3], a, [0, 0, 0, 0], r1, r2,
self.query), 0,
"No clicks, outcome should be 0.")
o = hpiIs.infer_outcome([1, 0, 3, 2], a, [1, 0, 0, 0], r1, r2,
self.query)
self.assertEquals(
o, -1, "Same original and target pair. "
"Ranker 1 should win (o = %d)." % o)
test_r1 = ProbabilisticRankingFunction(3, self.weights_1)
a = ([0, 1, 1, 0], r1, test_r1)
o = hpiIs.infer_outcome([1, 0, 3, 2], a, [1, 0, 0, 0], r2, r1,
self.query)
self.assertAlmostEquals(
o, -0.1250, 4, "Different original pair. "
"Ranker 1 should win (o = %.4f)." % o)
def testHistProbabilisticInterleave(self):
r1 = ProbabilisticRankingFunction(3, self.weights_1)
r2 = ProbabilisticRankingFunction(3, self.weights_2)
hpi = HistProbabilisticInterleave(None)
a = ([0, 1, 1, 0], r1, r2)
self.assertEqual(
hpi.infer_outcome([0, 1, 2, 3], a, [0, 0, 0, 0], r1, r2,
self.query), 0,
"No clicks, outcome should be 0.")
o = hpi.infer_outcome([1, 0, 3, 2], a, [1, 0, 0, 0], r1, r2,
self.query)
self.assertAlmostEquals(
o, -0.0486, 4, "Same target as original "
"rankers. Ranker 1 should win (o = %.4f)." % o)
o = hpi.infer_outcome([1, 0, 3, 2], a, [1, 0, 0, 0], r2, r1,
self.query)
self.assertAlmostEquals(
o, 0.0486, 4, "Target rankers switched. "
"Ranker 2 should win (o = %.4f)." % o)
test_r1 = ProbabilisticRankingFunction(3, self.weights_1)
a = ([0, 1, 1, 0], r1, test_r1)
o = hpi.infer_outcome([1, 0, 3, 2], a, [1, 0, 0, 0], r2, r1,
self.query)
self.assertAlmostEquals(
o, 0.1542, 4, "Same original ranker. "
"Ranker 2 should win (o = %.4f)." % o)
def testInferOutcomeUnbiased(self):
r1 = ProbabilisticRankingFunction(1, self.weights_1)
r2 = ProbabilisticRankingFunction(1, self.weights_2)
epi = ExploitativeProbabilisticInterleave("--exploration_rate=0.1")
outcome = epi.infer_outcome([1, 0, 3, 2], (None, r1, r2), [0, 1, 0, 0],
self.query)
self.assertAlmostEquals(0.03581, outcome, 8,
"Obtained outcome = %.8f" % outcome)
def testGetSourceProbabilityOfList(self):
r1 = ProbabilisticRankingFunction(1, self.weights_1)
r2 = ProbabilisticRankingFunction(1, self.weights_2)
# with exploration rate 0.5
epi = ExploitativeProbabilisticInterleave("--exploration_rate=0.5")
p = epi._get_source_probability_of_list([1, 0, 3, 2], (None, r1, r2),
self.query)
self.assertAlmostEquals(0.090916137, p, 8, "Obtained p = %.g" % p)
# with exploration rate 0.1
epi = ExploitativeProbabilisticInterleave("--exploration_rate=0.1")
p = epi._get_source_probability_of_list([1, 0, 3, 2], (None, r1, r2),
self.query)
self.assertAlmostEquals(0.073751736, p, 8, "Obtained p = %.g" % p)
def testExploitativeProbabilisticInterleaveInterleave(self):
r1 = ProbabilisticRankingFunction(1, self.weights_1)
r2 = ProbabilisticRankingFunction(1, self.weights_2)
epi = ExploitativeProbabilisticInterleave("--exploration_rate=0.5")
r1.init_ranking(self.query)
r2.init_ranking(self.query)
(l, (r1_ret, r2_ret)) = epi.interleave(r1, r2, self.query, 4)
self.assertEqual(r1, r1_ret, "r1 is just passed through.")
self.assertEqual(r2, r2_ret, "r2 is just passed through.")
self.assertEqual(len(l), 4, "interleave produces a list of length 4.")
self.assertTrue(0 in l, "document 0 is in l.")
self.assertTrue(1 in l, "document 0 is in l.")
self.assertTrue(2 in l, "document 0 is in l.")
self.assertTrue(3 in l, "document 0 is in l.")
observed_l = {}
for _ in range(0, 100):
(l, (r1_ret, r2_ret)) = epi.interleave(r1, r2, self.query, 4)
l_str = " ".join([str(docid) for docid in l])
if not l_str in observed_l:
observed_l[l_str] = 1
else:
observed_l[l_str] += 1
self.assertIn("0 1 2 3", observed_l, "List was observed: 0 1 2 3.")
self.assertIn("1 0 3 2", observed_l, "List was observed: 0 1 2 3.")
self.assertIn("3 1 2 0", observed_l, "List was observed: 0 1 2 3.")
self.assertIn("2 1 0 3", observed_l, "List was observed: 0 1 2 3.")
def testExploitativeProbabilisticInterleave(self):
r1 = ProbabilisticRankingFunction(1, self.weights_1)
r2 = ProbabilisticRankingFunction(1, self.weights_2)
r1.init_ranking(self.query)
r2.init_ranking(self.query)
epi = ExploitativeProbabilisticInterleave("--exploration_rate=0.5")
(docids, probs) = epi._get_document_distribution(r1, r2)
exp_docids = [1, 0, 3, 2]
exp_probs = [0.36, 0.3, 0.2, 0.14]
self._prob_doc_test_helper(docids, exp_docids, probs, exp_probs)
def testExploitativeProbabilisticInterleaveExploit(self):
r1 = ProbabilisticRankingFunction(1, self.weights_1)
r2 = ProbabilisticRankingFunction(1, self.weights_2)
# exploration rate = 0.1
epi = ExploitativeProbabilisticInterleave("--exploration_rate=0.1")
r1.init_ranking(self.query)
r2.init_ranking(self.query)
(docids, probs) = epi._get_document_distribution(r1, r2)
exp_docids = [1, 3, 2, 0]
exp_probs = [0.456, 0.232, 0.156, 0.156]
self._prob_doc_test_helper(docids, exp_docids, probs, exp_probs)
# exploration rate = 0.0
epi = ExploitativeProbabilisticInterleave("--exploration_rate=0.0")
r1.init_ranking(self.query)
r2.init_ranking(self.query)
(docids, probs) = epi._get_document_distribution(r1, r2)
exp_docids = [1, 3, 2, 0]
exp_probs = [0.48, 0.24, 0.16, 0.12]
self._prob_doc_test_helper(docids, exp_docids, probs, exp_probs)
def testExploitativeProbabilisticInterleaveInterleave(self):
r1 = ProbabilisticRankingFunction(1, self.weights_1)
r2 = ProbabilisticRankingFunction(1, self.weights_2)
epi = ExploitativeProbabilisticInterleave("--exploration_rate=0.5")
r1.init_ranking(self.query)
r2.init_ranking(self.query)
(l, (r1_ret, r2_ret)) = epi.interleave(r1, r2, self.query, 4)
self.assertEqual(r1, r1_ret, "r1 is just passed through.")
self.assertEqual(r2, r2_ret, "r2 is just passed through.")
self.assertEqual(len(l), 4, "interleave produces a list of length 4.")
self.assertTrue(0 in l, "document 0 is in l.")
self.assertTrue(1 in l, "document 0 is in l.")
self.assertTrue(2 in l, "document 0 is in l.")
self.assertTrue(3 in l, "document 0 is in l.")
observed_l = {}
for _ in range(0, 100):
(l, (r1_ret, r2_ret)) = epi.interleave(r1, r2, self.query, 4)
l_str = " ".join([str(docid) for docid in l])
if not l_str in observed_l:
observed_l[l_str] = 1
else:
observed_l[l_str] += 1
self.assertIn("0 1 2 3", observed_l, "List was observed: 0 1 2 3.")
self.assertIn("1 0 3 2", observed_l, "List was observed: 0 1 2 3.")
self.assertIn("3 1 2 0", observed_l, "List was observed: 0 1 2 3.")
self.assertIn("2 1 0 3", observed_l, "List was observed: 0 1 2 3.")
def testExploitativeProbabilisticInterleave(self):
r1 = ProbabilisticRankingFunction(1, self.weights_1)
r2 = ProbabilisticRankingFunction(1, self.weights_2)
r1.init_ranking(self.query)
r2.init_ranking(self.query)
epi = ExploitativeProbabilisticInterleave("--exploration_rate=0.5")
(docids, probs) = epi._get_document_distribution(r1, r2)
exp_docids = [1, 0, 3, 2]
exp_probs = [0.36, 0.3, 0.2, 0.14]
self._prob_doc_test_helper(docids, exp_docids, probs, exp_probs)
def testExploitativeProbabilisticInterleaveTwoDocs(self):
# prepare rankers
r1 = ProbabilisticRankingFunction(1, self.weights_1)
r2 = ProbabilisticRankingFunction(1, self.weights_2)
r1.init_ranking(self.query)
r2.init_ranking(self.query)
r1.rm_document(1)
r2.rm_document(1)
r1.rm_document(3)
r2.rm_document(3)
# test after 1 and 3 were removed
epi = ExploitativeProbabilisticInterleave("--exploration_rate=0.5")
(docids, probs) = epi._get_document_distribution(r1, r2)
exp_docids = [0, 2]
exp_probs = [0.61428571, 0.38571429]
self._prob_doc_test_helper(docids, exp_docids, probs, exp_probs)
def testExploitativeProbabilisticInterleaveThreeDocs(self):
epi = ExploitativeProbabilisticInterleave("--exploration_rate=0.5")
# prepare rankers
r1 = ProbabilisticRankingFunction(1, self.weights_1)
r2 = ProbabilisticRankingFunction(1, self.weights_2)
r1.init_ranking(self.query)
r2.init_ranking(self.query)
r1.rm_document(0)
r2.rm_document(0)
# test after document 0 was removed
(docids, probs) = epi._get_document_distribution(r1, r2)
exp_docids = [1, 3, 2]
exp_probs = [0.5034965, 0.29020979, 0.20629371]
self._prob_doc_test_helper(docids, exp_docids, probs, exp_probs)
# prepare rankers
r1.init_ranking(self.query)
r2.init_ranking(self.query)
r1.rm_document(3)
r2.rm_document(3)
# test after document 3 was removed
(docids, probs) = epi._get_document_distribution(r1, r2)
exp_docids = [1, 0, 2]
exp_probs = [0.45864662, 0.36466165, 0.17669173]
self._prob_doc_test_helper(docids, exp_docids, probs, exp_probs)
def testExploitativeProbabilisticInterleaveExploit(self):
r1 = ProbabilisticRankingFunction(1, self.weights_1)
r2 = ProbabilisticRankingFunction(1, self.weights_2)
# exploration rate = 0.1
epi = ExploitativeProbabilisticInterleave("--exploration_rate=0.1")
r1.init_ranking(self.query)
r2.init_ranking(self.query)
(docids, probs) = epi._get_document_distribution(r1, r2)
exp_docids = [1, 3, 2, 0]
exp_probs = [0.456, 0.232, 0.156, 0.156]
self._prob_doc_test_helper(docids, exp_docids, probs, exp_probs)
# exploration rate = 0.0
epi = ExploitativeProbabilisticInterleave("--exploration_rate=0.0")
r1.init_ranking(self.query)
r2.init_ranking(self.query)
(docids, probs) = epi._get_document_distribution(r1, r2)
exp_docids = [1, 3, 2, 0]
exp_probs = [0.48, 0.24, 0.16, 0.12]
self._prob_doc_test_helper(docids, exp_docids, probs, exp_probs)
def testExploitativeProbabilisticInterleaveTwoDocs(self):
# prepare rankers
r1 = ProbabilisticRankingFunction(1, self.weights_1)
r2 = ProbabilisticRankingFunction(1, self.weights_2)
r1.init_ranking(self.query)
r2.init_ranking(self.query)
r1.rm_document(1)
r2.rm_document(1)
r1.rm_document(3)
r2.rm_document(3)
# test after 1 and 3 were removed
epi = ExploitativeProbabilisticInterleave("--exploration_rate=0.5")
(docids, probs) = epi._get_document_distribution(r1, r2)
exp_docids = [0, 2]
exp_probs = [0.61428571, 0.38571429]
self._prob_doc_test_helper(docids, exp_docids, probs, exp_probs)
def testExploitativeProbabilisticInterleaveThreeDocs(self):
epi = ExploitativeProbabilisticInterleave("--exploration_rate=0.5")
# prepare rankers
r1 = ProbabilisticRankingFunction(1, self.weights_1)
r2 = ProbabilisticRankingFunction(1, self.weights_2)
r1.init_ranking(self.query)
r2.init_ranking(self.query)
r1.rm_document(0)
r2.rm_document(0)
# test after document 0 was removed
(docids, probs) = epi._get_document_distribution(r1, r2)
exp_docids = [1, 3, 2]
exp_probs = [0.5034965, 0.29020979, 0.20629371]
self._prob_doc_test_helper(docids, exp_docids, probs, exp_probs)
# prepare rankers
r1.init_ranking(self.query)
r2.init_ranking(self.query)
r1.rm_document(3)
r2.rm_document(3)
# test after document 3 was removed
(docids, probs) = epi._get_document_distribution(r1, r2)
exp_docids = [1, 0, 2]
exp_probs = [0.45864662, 0.36466165, 0.17669173]
self._prob_doc_test_helper(docids, exp_docids, probs, exp_probs)
def testProbabilisticInterleave(self):
pi = ProbabilisticInterleave(None)
r1 = ProbabilisticRankingFunction(3, self.weights_1)
r2 = ProbabilisticRankingFunction(3, self.weights_2)
context = (None, r1, r2)
# test get_probability_of_list
p = pi.get_probability_of_list([1, 0, 3, 2], context, self.query)
self.assertAlmostEquals(
p, 0.182775, 6, "Probability of the most "
"likely list. p = %.6f" % p)
# test a few possible interleavings
test_lists = {
"0,1,2,3": 0,
"0,1,3,2": 0,
"0,2,1,3": 0,
"0,2,3,1": 0,
"0,3,1,2": 0,
"0,3,2,1": 0,
"1,0,2,3": 0,
"1,0,3,2": 0,
"1,2,0,3": 0,
"1,2,3,0": 0,
"1,3,0,2": 0,
"1,3,2,0": 0,
"2,0,1,3": 0,
"2,0,3,1": 0,
"2,1,0,3": 0,
"2,1,3,0": 0,
"2,3,0,1": 0,
"2,3,1,0": 0,
"3,0,1,2": 0,
"3,0,2,1": 0,
"3,1,0,2": 0,
"3,1,2,0": 0,
"3,2,0,1": 0,
"3,2,1,0": 0
}
trials = 0
MAX_TRIALS = 100000
while trials < MAX_TRIALS and 0 in test_lists.values():
trials += 1
(l, _) = pi.interleave(r1, r2, self.query, 10)
list_str = ",".join(str(a) for a in l.tolist())
self.assertIn(list_str, test_lists.keys())
test_lists[list_str] += 1
for list_str, count in test_lists.items():
self.assertNotEqual(0, count,
"Interleave failed for: %s" % list_str)
# test interleaving outcomes
self.assertEqual(
pi.infer_outcome([0, 1, 2, 3], context, [0, 0, 0, 0], self.query),
0, "No clicks, outcome should be 0.")
o = pi.infer_outcome([1, 0, 3, 2], context, [1, 0, 0, 0], self.query)
self.assertAlmostEquals(o, -0.0486, 4,
"Ranker 1 should win (o = %.4f)." % o)
o = pi.infer_outcome([0, 1, 3, 2], context, [1, 0, 0, 0], self.query)
self.assertAlmostEquals(o, 0.0606, 4,
"Ranker 2 should win (o = %.4f)." % o)
# from the example in CIKM 2011
weight_str_1 = "0 0 1 0 -1 0"
weights_1 = np.asarray([float(x) for x in weight_str_1.split()])
weight_str_2 = "1 0 0 0 -1 0"
weights_2 = np.asarray([float(x) for x in weight_str_2.split()])
r1 = ProbabilisticRankingFunction(3, weights_1)
r2 = ProbabilisticRankingFunction(3, weights_2)
context = (None, r2, r1)
o = pi.infer_outcome([0, 1, 2, 3], context, [0, 1, 1, 0], self.query)
self.assertAlmostEquals(o, 0.0046, 4,
"Ranker 2 should win again (o = %.4f)." % o)
# click on one before last document
o = pi.infer_outcome([3, 1, 0, 2], context, [0, 0, 1, 0], self.query)
self.assertAlmostEquals(
o, -0.0496, 4,
"Ranker 1 should win with click on doc 0 (o = %.4f)." % o)
# click on last document
o = pi.infer_outcome([3, 1, 2, 0], context, [0, 0, 0, 1], self.query)
self.assertAlmostEquals(o, 0.0, 4,
"Tie for click on last doc (o = %.4f)." % o)