def testBalancedInterleave(self): bi = BalancedInterleave() r1 = DeterministicRankingFunction(None, self.weights_1) r2 = DeterministicRankingFunction(None, self.weights_2) (interleaved_list, assignments) = bi.interleave(r1, r2, self.query, 10) self.assertIn(interleaved_list.tolist(), [[0, 1, 2, 3], [1, 0, 2, 3], [0, 1, 3, 2], [1, 0, 3, 2]]) self.assertEqual(assignments[0].tolist(), [1, 3, 2, 0]) self.assertEqual(assignments[1].tolist(), [0, 1, 3, 2]) o = bi.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]], [1, 0, 0, 0], self.query) self.assertEqual(o, -1, "l1 should win (1), o = %g" % o) o = bi.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]], [1, 0, 1, 0], self.query) self.assertEqual(o, -1, "l1 should win (2), o = %g" % o) o = bi.infer_outcome([1, 0, 3, 2], [[1, 2, 3, 0], [0, 1, 3, 2]], [1, 0, 1, 0], self.query) self.assertEqual(o, 0, "The rankers should tie (1), o = %g" % o) o = bi.infer_outcome([0, 1, 2, 3], [[0, 1, 2, 3], [1, 2, 3, 0]], [0, 1, 0, 1], self.query) self.assertEqual(o, 1, "l1 should win, o = %g" % o) o = bi.infer_outcome([1, 0, 2, 3], [[0, 1, 2, 3], [1, 2, 3, 0]], [0, 1, 0, 1], self.query) self.assertEqual(o, 0, "The rankers should tie (2), o = %g" % o) o = bi.infer_outcome([0, 2, 1, 3], [[3, 0, 1, 2], [1, 3, 2, 0]], [1, 0, 1, 0], self.query) self.assertEqual(o, -1, "l1 should win (3), o = %g" % o) o = bi.infer_outcome([0, 2, 1, 3], [[3, 0, 1, 2], [4, 3, 2, 0]], [1, 0, 1, 0], self.query) self.assertEqual(o, -1, "l1 should win (4), o = %g" % o)
def testHistBalancedInterleave(self): hbi = HistBalancedInterleave() r1 = DeterministicRankingFunction(None, self.weights_1) r1_test = DeterministicRankingFunction(None, self.weights_1) r2 = DeterministicRankingFunction(None, self.weights_2) self.assertEqual( hbi._get_assignment(r1, r1_test, self.query, 4)[0].tolist(), [1, 3, 2, 0]) self.assertEqual( hbi._get_assignment(r1, r1_test, self.query, 4)[1].tolist(), [1, 3, 2, 0]) # r1 self.assertEqual( hbi._get_assignment(r1, r2, self.query, 4)[1].tolist(), [0, 1, 3, 2]) # r2 o = hbi.infer_outcome([1, 0, 3, 2], ["src a is ignored"], [1, 0, 0, 0], r1, r2, self.query) self.assertEqual(o, -1, "Same as original, l1 should win, o = %g" % o) o = hbi.infer_outcome([1, 0, 3, 2], ["src a is ignored"], [1, 0, 0, 0], r2, r1, self.query) self.assertEqual( o, 1, "Different from original, l2 should win, " "o = %g" % o) o = hbi.infer_outcome([1, 0, 3, 2], ["src a is ignored"], [1, 0, 0, 0], r1_test, r1, self.query) self.assertEqual(o, 0, "Same ranking - tie (1), o = %g" % o) o = hbi.infer_outcome([2, 0, 3, 1], ["src a is ignored"], [1, 1, 0, 0], r1, r2, self.query) self.assertEqual(o, 0, "Same ranking - tie (2), o = %g" % o) o = hbi.infer_outcome([2, 0, 3, 4], ["src a is ignored"], [1, 1, 0, 0], r1, r2, self.query) self.assertEqual(o, 0, "Same ranking - tie (3), o = %g" % o)
def testProbabilisticInterleaveWithDeterministicRankers(self): pi = ProbabilisticInterleave(None) # test a few possible interleavings r1 = DeterministicRankingFunction(None, self.weights_1) r2 = DeterministicRankingFunction(None, self.weights_2) test_lists = {"0,1,3,2": 0, "1,0,3,2": 0, "1,3,0,2": 0, "1,3,2,0": 0} trials = 0 MAX_TRIALS = 10000 while trials < MAX_TRIALS and 0 in test_lists.values(): trials += 1 (l, a) = pi.interleave(r1, r2, self.query, 10) list_str = ",".join(str(a) for a in l.tolist()) self.assertIn(list_str, test_lists.keys()) test_lists[list_str] += 1 for list_str, count in test_lists.items(): self.assertNotEqual(0, count, "Interleave failed for: %s" % list_str) # test interleaving outcomes context = (None, r1, r2) self.assertEqual( pi.infer_outcome([0, 1, 2, 3], context, [0, 0, 0, 0], self.query), 0, "No clicks, outcome should be 0.") self.assertEqual( pi.infer_outcome([0, 1, 2, 3], context, [1, 0, 0, 0], self.query), 0, "No possible assignment, outcome should be 0.") o = pi.infer_outcome([1, 0, 3, 2], context, [1, 0, 0, 0], self.query) self.assertAlmostEquals(o, -0.0625, 4, "Ranker 1 should win (o = %.4f)." % o) o = pi.infer_outcome([0, 1, 3, 2], context, [1, 0, 0, 0], self.query) self.assertAlmostEquals(o, 0.0625, 4, "Ranker 2 should win (o = %.4f)." % o) # test get_probability_of_list p = pi.get_probability_of_list([1, 0, 3, 2], context, self.query) self.assertEqual(p, 0.25, "Probability of the most " "likely list. p = %g" % p)
def testTeamDraftInterleave(self): td = TeamDraft(None) r1 = DeterministicRankingFunction(None, self.weights_1) r2 = DeterministicRankingFunction(None, self.weights_2) (interleaved_list, assignments) = td.interleave(r1, r2, self.query, 10) self.assertIn(interleaved_list.tolist(), [[0, 1, 2, 3], [1, 0, 2, 3], [0, 1, 3, 2], [1, 0, 3, 2]]) self.assertIn(assignments.tolist(), [[0, 1, 0, 1], [1, 0, 1, 0], [1, 0, 0, 1], [0, 1, 1, 0]])
def testHistTeamDraft_getPossibleAssignment(self): r1 = DeterministicRankingFunction(None, self.weights_1) r2 = DeterministicRankingFunction(None, self.weights_2) htd = HistTeamDraft(None) l = [0, 1, 3, 2] self.assertIn(htd._get_possible_assignment(l, r1, r2, self.query), [[1, 0, 0, 1], [1, 0, 1, 0]]) l = [1, 0, 3, 2] self.assertIn(htd._get_possible_assignment(l, r1, r2, self.query), [[0, 1, 0, 1], [0, 1, 1, 0]]) l = [1, 0, 2, 3] self.assertEquals(htd._get_possible_assignment(l, r1, r2, self.query), None)
def testDocumentConstraints(self): dc = DocumentConstraints() r1 = DeterministicRankingFunction(None, self.weights_1) r2 = DeterministicRankingFunction(None, self.weights_2) (interleaved_list, assignments) = dc.interleave(r1, r2, self.query, 10) self.assertIn(interleaved_list.tolist(), [[0, 1, 2, 3], [1, 0, 2, 3], [0, 1, 3, 2], [1, 0, 3, 2]]) self.assertIn(assignments[0].tolist(), [[1, 2, 3, 0], [1, 3, 2, 0]]) self.assertIn(assignments[1].tolist(), [[0, 1, 2, 3], [0, 1, 3, 2]]) o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]], [1, 0, 0, 0], self.query) self.assertEqual(o, -1, "l1 should win (1), o = %g" % o) o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]], [0, 0, 0, 1], self.query) self.assertEqual(o, -1, "l1 should win (2), o = %g" % o) o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]], [0, 1, 0, 0], self.query) self.assertEqual(o, 1, "l2 should win (1), o = %g" % o) o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 2, 3], [0, 1, 3, 2]], [0, 1, 0, 0], self.query) # constraints: 0 > 1, 0 > 3 self.assertEqual(o, 1, "l2 should win (2), o = %g" % o) o = dc.infer_outcome([1, 0, 3, 2], [[1, 2, 0, 3], [1, 0, 2, 3]], [0, 1, 1, 0], self.query) # constraints: 0 > 1, 3 > 1, 0 > 2, 3 > 2 self.assertEqual(o, 1, "l2 should win (3), o = %g" % o) o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]], [0, 0, 0, 0], self.query) self.assertEqual(o, 0, "No winner when there are no clicks o = %g" % o) o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]], [1, 1, 1, 1], self.query) self.assertEqual(o, 0, "No winner when all are clicked o = %g" % o) dc = DocumentConstraints("--constraints 1") o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 2, 3], [3, 0, 1, 2]], [0, 1, 0, 0], self.query) # constraint: 0 > 1 self.assertEqual(o, 1, "l2 should win with one constraint, o = %g" % o) dc = DocumentConstraints("--constraints 2") o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 2, 3], [3, 0, 1, 2]], [0, 1, 0, 0], self.query) self.assertEqual(o, 0, "Tie with two constraint types (1), o = %g" % o) o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 2, 3], [1, 2, 0, 3]], [0, 1, 1, 0], self.query) # constraints: 0 > 1, 3 > 1, 3 > 2 self.assertEqual(o, 0, "Tie with two constraint types (2), o = %g" % o) o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 4, 3], [1, 0, 3, 2]], [0, 1, 1, 0], self.query) self.assertEqual(o, 0, "Tie with two constraint types (3), o = %g" % o) o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 4, 3], [1, 0, 2, 3]], [0, 1, 1, 0], self.query) self.assertEqual(o, -1, "l1 should win with two constr., o = %g" % o)
def testHistTeamDraft(self): r1 = DeterministicRankingFunction(None, self.weights_1) r2 = DeterministicRankingFunction(None, self.weights_2) interleaved_list = [0, 1, 3, 2] htd = HistTeamDraft() self.assertEqual( htd.infer_outcome(interleaved_list, None, [0, 0, 0, 0], r1, r2, self.query), 0, "No clicks.") self.assertEqual( htd.infer_outcome(interleaved_list, None, [1, 0, 0, 0], r1, r2, self.query), 1, "Target rankers" " are the same as the original rankers, so ranker 2 has to win.") self.assertEqual( htd.infer_outcome(interleaved_list, None, [1, 0, 0, 0], r2, r1, self.query), -1, "Target rankers" " are switched, so ranker 1 has to win.")
def testHistTeamDraft_getPossibleAssignment_randomization(self): r1 = DeterministicRankingFunction(None, self.weights_1) r2 = DeterministicRankingFunction(None, self.weights_2) htd = HistTeamDraft(None) l = [0, 1, 3, 2] test_assignments = {"1,0,0,1": 0, "1,0,1,0": 0} trials = 0 MAX_TRIALS = 1000 while trials < MAX_TRIALS and 0 in test_assignments.values(): trials += 1 observed_assignment = ",".join( str(a) for a in htd._get_possible_assignment(l, r1, r2, self.query)) self.assertIn(observed_assignment, test_assignments.keys()) test_assignments[observed_assignment] += 1 for assignment, count in test_assignments.items(): self.assertNotEqual(0, count, "Test failed for: %s" % assignment)
def testHistDocumentConstraints(self): hdc = HistDocumentConstraints() r1 = DeterministicRankingFunction(None, self.weights_1) r2 = DeterministicRankingFunction(None, self.weights_2) # results in assignments l1 = [1, 2, 3, 0] or [1, 3, 2, 0] # and l2 = [0, 1, 2, 3] or [0, 1, 3, 2] o = hdc.infer_outcome([1, 0, 3, 2], None, [1, 0, 0, 0], r1, r2, self.query) self.assertEqual(o, -1, "l1 should win, o = %g" % o) o = hdc.infer_outcome([2, 1, 3, 0], None, [1, 0, 0, 0], r1, r2, self.query) self.assertEqual(o, 0, "No winner, both have 1 > 2 (1), o = %g" % o) o = hdc.infer_outcome([2, 1, 4, 0], None, [1, 0, 0, 0], r1, r2, self.query) self.assertEqual(o, 0, "No winner, both have 1 > 2 (2), o = %g" % o) o = hdc.infer_outcome([2, 1, 3, 0], None, [0, 0, 0, 0], r1, r2, self.query) self.assertEqual(o, 0, "No winner when none are clicked, o = %g" % o) o = hdc.infer_outcome([2, 1, 3, 0], None, [1, 1, 1, 1], r1, r2, self.query) self.assertEqual(o, 0, "No winner when all are clicked, o = %g" % o)