def testBalancedInterleave(self):
bi = BalancedInterleave()
r1 = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
(interleaved_list, assignments) = bi.interleave(r1, r2, self.query, 10)
self.assertIn(interleaved_list.tolist(),
[[0, 1, 2, 3], [1, 0, 2, 3], [0, 1, 3, 2], [1, 0, 3, 2]])
self.assertEqual(assignments[0].tolist(), [1, 3, 2, 0])
self.assertEqual(assignments[1].tolist(), [0, 1, 3, 2])
o = bi.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[1, 0, 0, 0], self.query)
self.assertEqual(o, -1, "l1 should win (1), o = %g" % o)
o = bi.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[1, 0, 1, 0], self.query)
self.assertEqual(o, -1, "l1 should win (2), o = %g" % o)
o = bi.infer_outcome([1, 0, 3, 2], [[1, 2, 3, 0], [0, 1, 3, 2]],
[1, 0, 1, 0], self.query)
self.assertEqual(o, 0, "The rankers should tie (1), o = %g" % o)
o = bi.infer_outcome([0, 1, 2, 3], [[0, 1, 2, 3], [1, 2, 3, 0]],
[0, 1, 0, 1], self.query)
self.assertEqual(o, 1, "l1 should win, o = %g" % o)
o = bi.infer_outcome([1, 0, 2, 3], [[0, 1, 2, 3], [1, 2, 3, 0]],
[0, 1, 0, 1], self.query)
self.assertEqual(o, 0, "The rankers should tie (2), o = %g" % o)
o = bi.infer_outcome([0, 2, 1, 3], [[3, 0, 1, 2], [1, 3, 2, 0]],
[1, 0, 1, 0], self.query)
self.assertEqual(o, -1, "l1 should win (3), o = %g" % o)
o = bi.infer_outcome([0, 2, 1, 3], [[3, 0, 1, 2], [4, 3, 2, 0]],
[1, 0, 1, 0], self.query)
self.assertEqual(o, -1, "l1 should win (4), o = %g" % o)
def testHistBalancedInterleave(self):
hbi = HistBalancedInterleave()
r1 = DeterministicRankingFunction(None, self.weights_1)
r1_test = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
self.assertEqual(
hbi._get_assignment(r1, r1_test, self.query, 4)[0].tolist(),
[1, 3, 2, 0])
self.assertEqual(
hbi._get_assignment(r1, r1_test, self.query, 4)[1].tolist(),
[1, 3, 2, 0]) # r1
self.assertEqual(
hbi._get_assignment(r1, r2, self.query, 4)[1].tolist(),
[0, 1, 3, 2]) # r2
o = hbi.infer_outcome([1, 0, 3, 2], ["src a is ignored"], [1, 0, 0, 0],
r1, r2, self.query)
self.assertEqual(o, -1, "Same as original, l1 should win, o = %g" % o)
o = hbi.infer_outcome([1, 0, 3, 2], ["src a is ignored"], [1, 0, 0, 0],
r2, r1, self.query)
self.assertEqual(
o, 1, "Different from original, l2 should win, "
"o = %g" % o)
o = hbi.infer_outcome([1, 0, 3, 2], ["src a is ignored"], [1, 0, 0, 0],
r1_test, r1, self.query)
self.assertEqual(o, 0, "Same ranking - tie (1), o = %g" % o)
o = hbi.infer_outcome([2, 0, 3, 1], ["src a is ignored"], [1, 1, 0, 0],
r1, r2, self.query)
self.assertEqual(o, 0, "Same ranking - tie (2), o = %g" % o)
o = hbi.infer_outcome([2, 0, 3, 4], ["src a is ignored"], [1, 1, 0, 0],
r1, r2, self.query)
self.assertEqual(o, 0, "Same ranking - tie (3), o = %g" % o)
def testProbabilisticInterleaveWithDeterministicRankers(self):
pi = ProbabilisticInterleave(None)
# test a few possible interleavings
r1 = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
test_lists = {"0,1,3,2": 0, "1,0,3,2": 0, "1,3,0,2": 0, "1,3,2,0": 0}
trials = 0
MAX_TRIALS = 10000
while trials < MAX_TRIALS and 0 in test_lists.values():
trials += 1
(l, a) = pi.interleave(r1, r2, self.query, 10)
list_str = ",".join(str(a) for a in l.tolist())
self.assertIn(list_str, test_lists.keys())
test_lists[list_str] += 1
for list_str, count in test_lists.items():
self.assertNotEqual(0, count,
"Interleave failed for: %s" % list_str)
# test interleaving outcomes
context = (None, r1, r2)
self.assertEqual(
pi.infer_outcome([0, 1, 2, 3], context, [0, 0, 0, 0], self.query),
0, "No clicks, outcome should be 0.")
self.assertEqual(
pi.infer_outcome([0, 1, 2, 3], context, [1, 0, 0, 0], self.query),
0, "No possible assignment, outcome should be 0.")
o = pi.infer_outcome([1, 0, 3, 2], context, [1, 0, 0, 0], self.query)
self.assertAlmostEquals(o, -0.0625, 4,
"Ranker 1 should win (o = %.4f)." % o)
o = pi.infer_outcome([0, 1, 3, 2], context, [1, 0, 0, 0], self.query)
self.assertAlmostEquals(o, 0.0625, 4,
"Ranker 2 should win (o = %.4f)." % o)
# test get_probability_of_list
p = pi.get_probability_of_list([1, 0, 3, 2], context, self.query)
self.assertEqual(p, 0.25, "Probability of the most "
"likely list. p = %g" % p)
def testTeamDraftInterleave(self):
td = TeamDraft(None)
r1 = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
(interleaved_list, assignments) = td.interleave(r1, r2, self.query, 10)
self.assertIn(interleaved_list.tolist(),
[[0, 1, 2, 3], [1, 0, 2, 3], [0, 1, 3, 2], [1, 0, 3, 2]])
self.assertIn(assignments.tolist(),
[[0, 1, 0, 1], [1, 0, 1, 0], [1, 0, 0, 1], [0, 1, 1, 0]])
def testHistTeamDraft_getPossibleAssignment(self):
r1 = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
htd = HistTeamDraft(None)
l = [0, 1, 3, 2]
self.assertIn(htd._get_possible_assignment(l, r1, r2, self.query),
[[1, 0, 0, 1], [1, 0, 1, 0]])
l = [1, 0, 3, 2]
self.assertIn(htd._get_possible_assignment(l, r1, r2, self.query),
[[0, 1, 0, 1], [0, 1, 1, 0]])
l = [1, 0, 2, 3]
self.assertEquals(htd._get_possible_assignment(l, r1, r2, self.query),
None)
def testDocumentConstraints(self):
dc = DocumentConstraints()
r1 = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
(interleaved_list, assignments) = dc.interleave(r1, r2, self.query, 10)
self.assertIn(interleaved_list.tolist(),
[[0, 1, 2, 3], [1, 0, 2, 3], [0, 1, 3, 2], [1, 0, 3, 2]])
self.assertIn(assignments[0].tolist(), [[1, 2, 3, 0], [1, 3, 2, 0]])
self.assertIn(assignments[1].tolist(), [[0, 1, 2, 3], [0, 1, 3, 2]])
o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[1, 0, 0, 0], self.query)
self.assertEqual(o, -1, "l1 should win (1), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[0, 0, 0, 1], self.query)
self.assertEqual(o, -1, "l1 should win (2), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[0, 1, 0, 0], self.query)
self.assertEqual(o, 1, "l2 should win (1), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 2, 3], [0, 1, 3, 2]],
[0, 1, 0, 0], self.query)
# constraints: 0 > 1, 0 > 3
self.assertEqual(o, 1, "l2 should win (2), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 2, 0, 3], [1, 0, 2, 3]],
[0, 1, 1, 0], self.query)
# constraints: 0 > 1, 3 > 1, 0 > 2, 3 > 2
self.assertEqual(o, 1, "l2 should win (3), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[0, 0, 0, 0], self.query)
self.assertEqual(o, 0, "No winner when there are no clicks o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[1, 1, 1, 1], self.query)
self.assertEqual(o, 0, "No winner when all are clicked o = %g" % o)
dc = DocumentConstraints("--constraints 1")
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 2, 3], [3, 0, 1, 2]],
[0, 1, 0, 0], self.query)
# constraint: 0 > 1
self.assertEqual(o, 1, "l2 should win with one constraint, o = %g" % o)
dc = DocumentConstraints("--constraints 2")
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 2, 3], [3, 0, 1, 2]],
[0, 1, 0, 0], self.query)
self.assertEqual(o, 0, "Tie with two constraint types (1), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 2, 3], [1, 2, 0, 3]],
[0, 1, 1, 0], self.query)
# constraints: 0 > 1, 3 > 1, 3 > 2
self.assertEqual(o, 0, "Tie with two constraint types (2), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 4, 3], [1, 0, 3, 2]],
[0, 1, 1, 0], self.query)
self.assertEqual(o, 0, "Tie with two constraint types (3), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 4, 3], [1, 0, 2, 3]],
[0, 1, 1, 0], self.query)
self.assertEqual(o, -1, "l1 should win with two constr., o = %g" % o)
def testHistTeamDraft(self):
r1 = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
interleaved_list = [0, 1, 3, 2]
htd = HistTeamDraft()
self.assertEqual(
htd.infer_outcome(interleaved_list, None, [0, 0, 0, 0], r1, r2,
self.query), 0, "No clicks.")
self.assertEqual(
htd.infer_outcome(interleaved_list, None, [1, 0, 0, 0], r1, r2,
self.query), 1, "Target rankers"
" are the same as the original rankers, so ranker 2 has to win.")
self.assertEqual(
htd.infer_outcome(interleaved_list, None, [1, 0, 0, 0], r2, r1,
self.query), -1, "Target rankers"
" are switched, so ranker 1 has to win.")
def testHistTeamDraft_getPossibleAssignment_randomization(self):
r1 = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
htd = HistTeamDraft(None)
l = [0, 1, 3, 2]
test_assignments = {"1,0,0,1": 0, "1,0,1,0": 0}
trials = 0
MAX_TRIALS = 1000
while trials < MAX_TRIALS and 0 in test_assignments.values():
trials += 1
observed_assignment = ",".join(
str(a)
for a in htd._get_possible_assignment(l, r1, r2, self.query))
self.assertIn(observed_assignment, test_assignments.keys())
test_assignments[observed_assignment] += 1
for assignment, count in test_assignments.items():
self.assertNotEqual(0, count, "Test failed for: %s" % assignment)
def testHistDocumentConstraints(self):
hdc = HistDocumentConstraints()
r1 = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
# results in assignments l1 = [1, 2, 3, 0] or [1, 3, 2, 0]
# and l2 = [0, 1, 2, 3] or [0, 1, 3, 2]
o = hdc.infer_outcome([1, 0, 3, 2], None, [1, 0, 0, 0], r1, r2,
self.query)
self.assertEqual(o, -1, "l1 should win, o = %g" % o)
o = hdc.infer_outcome([2, 1, 3, 0], None, [1, 0, 0, 0], r1, r2,
self.query)
self.assertEqual(o, 0, "No winner, both have 1 > 2 (1), o = %g" % o)
o = hdc.infer_outcome([2, 1, 4, 0], None, [1, 0, 0, 0], r1, r2,
self.query)
self.assertEqual(o, 0, "No winner, both have 1 > 2 (2), o = %g" % o)
o = hdc.infer_outcome([2, 1, 3, 0], None, [0, 0, 0, 0], r1, r2,
self.query)
self.assertEqual(o, 0, "No winner when none are clicked, o = %g" % o)
o = hdc.infer_outcome([2, 1, 3, 0], None, [1, 1, 1, 1], r1, r2,
self.query)
self.assertEqual(o, 0, "No winner when all are clicked, o = %g" % o)
