def testBalancedInterleave(self):
bi = BalancedInterleave()
r1 = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
(interleaved_list, assignments) = bi.interleave(r1, r2, self.query, 10)
self.assertIn(interleaved_list.tolist(),
[[0, 1, 2, 3], [1, 0, 2, 3], [0, 1, 3, 2], [1, 0, 3, 2]])
self.assertEqual(assignments[0].tolist(), [1, 3, 2, 0])
self.assertEqual(assignments[1].tolist(), [0, 1, 3, 2])
o = bi.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[1, 0, 0, 0], self.query)
self.assertEqual(o, -1, "l1 should win (1), o = %g" % o)
o = bi.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[1, 0, 1, 0], self.query)
self.assertEqual(o, -1, "l1 should win (2), o = %g" % o)
o = bi.infer_outcome([1, 0, 3, 2], [[1, 2, 3, 0], [0, 1, 3, 2]],
[1, 0, 1, 0], self.query)
self.assertEqual(o, 0, "The rankers should tie (1), o = %g" % o)
o = bi.infer_outcome([0, 1, 2, 3], [[0, 1, 2, 3], [1, 2, 3, 0]],
[0, 1, 0, 1], self.query)
self.assertEqual(o, 1, "l1 should win, o = %g" % o)
o = bi.infer_outcome([1, 0, 2, 3], [[0, 1, 2, 3], [1, 2, 3, 0]],
[0, 1, 0, 1], self.query)
self.assertEqual(o, 0, "The rankers should tie (2), o = %g" % o)
o = bi.infer_outcome([0, 2, 1, 3], [[3, 0, 1, 2], [1, 3, 2, 0]],
[1, 0, 1, 0], self.query)
self.assertEqual(o, -1, "l1 should win (3), o = %g" % o)
o = bi.infer_outcome([0, 2, 1, 3], [[3, 0, 1, 2], [4, 3, 2, 0]],
[1, 0, 1, 0], self.query)
self.assertEqual(o, -1, "l1 should win (4), o = %g" % o)
def testHistBalancedInterleave(self):
hbi = HistBalancedInterleave()
r1 = DeterministicRankingFunction(None, self.weights_1)
r1_test = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
self.assertEqual(
hbi._get_assignment(r1, r1_test, self.query, 4)[0].tolist(),
[1, 3, 2, 0])
self.assertEqual(
hbi._get_assignment(r1, r1_test, self.query, 4)[1].tolist(),
[1, 3, 2, 0]) # r1
self.assertEqual(
hbi._get_assignment(r1, r2, self.query, 4)[1].tolist(),
[0, 1, 3, 2]) # r2
o = hbi.infer_outcome([1, 0, 3, 2], ["src a is ignored"], [1, 0, 0, 0],
r1, r2, self.query)
self.assertEqual(o, -1, "Same as original, l1 should win, o = %g" % o)
o = hbi.infer_outcome([1, 0, 3, 2], ["src a is ignored"], [1, 0, 0, 0],
r2, r1, self.query)
self.assertEqual(
o, 1, "Different from original, l2 should win, "
"o = %g" % o)
o = hbi.infer_outcome([1, 0, 3, 2], ["src a is ignored"], [1, 0, 0, 0],
r1_test, r1, self.query)
self.assertEqual(o, 0, "Same ranking - tie (1), o = %g" % o)
o = hbi.infer_outcome([2, 0, 3, 1], ["src a is ignored"], [1, 1, 0, 0],
r1, r2, self.query)
self.assertEqual(o, 0, "Same ranking - tie (2), o = %g" % o)
o = hbi.infer_outcome([2, 0, 3, 4], ["src a is ignored"], [1, 1, 0, 0],
r1, r2, self.query)
self.assertEqual(o, 0, "Same ranking - tie (3), o = %g" % o)
def testProbabilisticInterleaveWithDeterministicRankers(self):
pi = ProbabilisticInterleave(None)
# test a few possible interleavings
r1 = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
test_lists = {"0,1,3,2": 0, "1,0,3,2": 0, "1,3,0,2": 0, "1,3,2,0": 0}
trials = 0
MAX_TRIALS = 10000
while trials < MAX_TRIALS and 0 in test_lists.values():
trials += 1
(l, a) = pi.interleave(r1, r2, self.query, 10)
list_str = ",".join(str(a) for a in l.tolist())
self.assertIn(list_str, test_lists.keys())
test_lists[list_str] += 1
for list_str, count in test_lists.items():
self.assertNotEqual(0, count,
"Interleave failed for: %s" % list_str)
# test interleaving outcomes
context = (None, r1, r2)
self.assertEqual(
pi.infer_outcome([0, 1, 2, 3], context, [0, 0, 0, 0], self.query),
0, "No clicks, outcome should be 0.")
self.assertEqual(
pi.infer_outcome([0, 1, 2, 3], context, [1, 0, 0, 0], self.query),
0, "No possible assignment, outcome should be 0.")
o = pi.infer_outcome([1, 0, 3, 2], context, [1, 0, 0, 0], self.query)
self.assertAlmostEquals(o, -0.0625, 4,
"Ranker 1 should win (o = %.4f)." % o)
o = pi.infer_outcome([0, 1, 3, 2], context, [1, 0, 0, 0], self.query)
self.assertAlmostEquals(o, 0.0625, 4,
"Ranker 2 should win (o = %.4f)." % o)
# test get_probability_of_list
p = pi.get_probability_of_list([1, 0, 3, 2], context, self.query)
self.assertEqual(p, 0.25, "Probability of the most "
"likely list. p = %g" % p)
def testTeamDraftInterleave(self):
td = TeamDraft(None)
r1 = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
(interleaved_list, assignments) = td.interleave(r1, r2, self.query, 10)
self.assertIn(interleaved_list.tolist(),
[[0, 1, 2, 3], [1, 0, 2, 3], [0, 1, 3, 2], [1, 0, 3, 2]])
self.assertIn(assignments.tolist(),
[[0, 1, 0, 1], [1, 0, 1, 0], [1, 0, 0, 1], [0, 1, 1, 0]])
def testHistTeamDraft_getPossibleAssignment(self):
r1 = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
htd = HistTeamDraft(None)
l = [0, 1, 3, 2]
self.assertIn(htd._get_possible_assignment(l, r1, r2, self.query),
[[1, 0, 0, 1], [1, 0, 1, 0]])
l = [1, 0, 3, 2]
self.assertIn(htd._get_possible_assignment(l, r1, r2, self.query),
[[0, 1, 0, 1], [0, 1, 1, 0]])
l = [1, 0, 2, 3]
self.assertEquals(htd._get_possible_assignment(l, r1, r2, self.query),
None)
def testDocumentConstraints(self):
dc = DocumentConstraints()
r1 = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
(interleaved_list, assignments) = dc.interleave(r1, r2, self.query, 10)
self.assertIn(interleaved_list.tolist(),
[[0, 1, 2, 3], [1, 0, 2, 3], [0, 1, 3, 2], [1, 0, 3, 2]])
self.assertIn(assignments[0].tolist(), [[1, 2, 3, 0], [1, 3, 2, 0]])
self.assertIn(assignments[1].tolist(), [[0, 1, 2, 3], [0, 1, 3, 2]])
o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[1, 0, 0, 0], self.query)
self.assertEqual(o, -1, "l1 should win (1), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[0, 0, 0, 1], self.query)
self.assertEqual(o, -1, "l1 should win (2), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[0, 1, 0, 0], self.query)
self.assertEqual(o, 1, "l2 should win (1), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 2, 3], [0, 1, 3, 2]],
[0, 1, 0, 0], self.query)
# constraints: 0 > 1, 0 > 3
self.assertEqual(o, 1, "l2 should win (2), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 2, 0, 3], [1, 0, 2, 3]],
[0, 1, 1, 0], self.query)
# constraints: 0 > 1, 3 > 1, 0 > 2, 3 > 2
self.assertEqual(o, 1, "l2 should win (3), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[0, 0, 0, 0], self.query)
self.assertEqual(o, 0, "No winner when there are no clicks o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[1, 1, 1, 1], self.query)
self.assertEqual(o, 0, "No winner when all are clicked o = %g" % o)
dc = DocumentConstraints("--constraints 1")
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 2, 3], [3, 0, 1, 2]],
[0, 1, 0, 0], self.query)
# constraint: 0 > 1
self.assertEqual(o, 1, "l2 should win with one constraint, o = %g" % o)
dc = DocumentConstraints("--constraints 2")
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 2, 3], [3, 0, 1, 2]],
[0, 1, 0, 0], self.query)
self.assertEqual(o, 0, "Tie with two constraint types (1), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 2, 3], [1, 2, 0, 3]],
[0, 1, 1, 0], self.query)
# constraints: 0 > 1, 3 > 1, 3 > 2
self.assertEqual(o, 0, "Tie with two constraint types (2), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 4, 3], [1, 0, 3, 2]],
[0, 1, 1, 0], self.query)
self.assertEqual(o, 0, "Tie with two constraint types (3), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 4, 3], [1, 0, 2, 3]],
[0, 1, 1, 0], self.query)
self.assertEqual(o, -1, "l1 should win with two constr., o = %g" % o)
def setUp(self):
# Create ranker to test with
ranker_args = ['3']
ranker_tie = 'random'
init_weights = 'random'
feature_count = 5
self.ranker = DRF(ranker_args,
ranker_tie,
feature_count,
sample=None,
init=init_weights)
# Create queries to test with
test_queries = """
1 qid:373 1:0.089908 2:0.531250 3:0.500000 4:0.500000 5:0.156538
0 qid:373 1:0.066055 2:0.171875 3:0.000000 4:0.250000 5:0.084715
0 qid:373 1:0.148624 2:0.015625 3:0.250000 4:0.250000 5:0.151013
0 qid:373 1:0.099083 2:0.250000 3:0.500000 4:0.750000 5:0.134438
0 qid:373 1:0.051376 2:0.078125 3:0.250000 4:0.250000 5:0.060773
0 qid:373 1:0.045872 2:1.000000 3:0.250000 4:0.250000 5:0.163904
"""
query_fh = cStringIO.StringIO(test_queries)
self.query = query.Queries(query_fh, feature_count)['373']
query_fh.close()
# Save the original ranking
self.ranker.init_ranking(self.query)
self.ranking = [
self.ranker.next() for _ in range(self.ranker.document_count())
]
def setUp(self):
# Create ranker to test with
ranker_args = ['3']
ranker_tie = 'random'
init_weights = 'random'
feature_count = 5
self.ranker = DRF(
ranker_args,
ranker_tie,
feature_count,
sample=None,
init=init_weights
)
# Create queries to test with
test_queries = """
1 qid:373 1:0.089908 2:0.531250 3:0.500000 4:0.500000 5:0.156538
0 qid:373 1:0.066055 2:0.171875 3:0.000000 4:0.250000 5:0.084715
0 qid:373 1:0.148624 2:0.015625 3:0.250000 4:0.250000 5:0.151013
0 qid:373 1:0.099083 2:0.250000 3:0.500000 4:0.750000 5:0.134438
0 qid:373 1:0.051376 2:0.078125 3:0.250000 4:0.250000 5:0.060773
0 qid:373 1:0.045872 2:1.000000 3:0.250000 4:0.250000 5:0.163904
"""
query_fh = cStringIO.StringIO(test_queries)
self.query = query.Queries(query_fh, feature_count)['373']
query_fh.close()
# Save the original ranking
self.ranker.init_ranking(self.query)
self.ranking = [
self.ranker.next() for _ in range(self.ranker.document_count())
]
def testHistTeamDraft(self):
r1 = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
interleaved_list = [0, 1, 3, 2]
htd = HistTeamDraft()
self.assertEqual(
htd.infer_outcome(interleaved_list, None, [0, 0, 0, 0], r1, r2,
self.query), 0, "No clicks.")
self.assertEqual(
htd.infer_outcome(interleaved_list, None, [1, 0, 0, 0], r1, r2,
self.query), 1, "Target rankers"
" are the same as the original rankers, so ranker 2 has to win.")
self.assertEqual(
htd.infer_outcome(interleaved_list, None, [1, 0, 0, 0], r2, r1,
self.query), -1, "Target rankers"
" are switched, so ranker 1 has to win.")
def testHistTeamDraft_getPossibleAssignment_randomization(self):
r1 = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
htd = HistTeamDraft(None)
l = [0, 1, 3, 2]
test_assignments = {"1,0,0,1": 0, "1,0,1,0": 0}
trials = 0
MAX_TRIALS = 1000
while trials < MAX_TRIALS and 0 in test_assignments.values():
trials += 1
observed_assignment = ",".join(
str(a)
for a in htd._get_possible_assignment(l, r1, r2, self.query))
self.assertIn(observed_assignment, test_assignments.keys())
test_assignments[observed_assignment] += 1
for assignment, count in test_assignments.items():
self.assertNotEqual(0, count, "Test failed for: %s" % assignment)
def testHistDocumentConstraints(self):
hdc = HistDocumentConstraints()
r1 = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
# results in assignments l1 = [1, 2, 3, 0] or [1, 3, 2, 0]
# and l2 = [0, 1, 2, 3] or [0, 1, 3, 2]
o = hdc.infer_outcome([1, 0, 3, 2], None, [1, 0, 0, 0], r1, r2,
self.query)
self.assertEqual(o, -1, "l1 should win, o = %g" % o)
o = hdc.infer_outcome([2, 1, 3, 0], None, [1, 0, 0, 0], r1, r2,
self.query)
self.assertEqual(o, 0, "No winner, both have 1 > 2 (1), o = %g" % o)
o = hdc.infer_outcome([2, 1, 4, 0], None, [1, 0, 0, 0], r1, r2,
self.query)
self.assertEqual(o, 0, "No winner, both have 1 > 2 (2), o = %g" % o)
o = hdc.infer_outcome([2, 1, 3, 0], None, [0, 0, 0, 0], r1, r2,
self.query)
self.assertEqual(o, 0, "No winner when none are clicked, o = %g" % o)
o = hdc.infer_outcome([2, 1, 3, 0], None, [1, 1, 1, 1], r1, r2,
self.query)
self.assertEqual(o, 0, "No winner when all are clicked, o = %g" % o)
class TestPerturbation(unittest.TestCase):
TRIES = 1000
MARGIN = 5 / TRIES
def setUp(self):
# Create ranker to test with
ranker_args = ['3']
ranker_tie = 'random'
init_weights = 'random'
feature_count = 5
self.ranker = DRF(ranker_args,
ranker_tie,
feature_count,
sample=None,
init=init_weights)
# Create queries to test with
test_queries = """
1 qid:373 1:0.089908 2:0.531250 3:0.500000 4:0.500000 5:0.156538
0 qid:373 1:0.066055 2:0.171875 3:0.000000 4:0.250000 5:0.084715
0 qid:373 1:0.148624 2:0.015625 3:0.250000 4:0.250000 5:0.151013
0 qid:373 1:0.099083 2:0.250000 3:0.500000 4:0.750000 5:0.134438
0 qid:373 1:0.051376 2:0.078125 3:0.250000 4:0.250000 5:0.060773
0 qid:373 1:0.045872 2:1.000000 3:0.250000 4:0.250000 5:0.163904
"""
query_fh = cStringIO.StringIO(test_queries)
self.query = query.Queries(query_fh, feature_count)['373']
query_fh.close()
# Save the original ranking
self.ranker.init_ranking(self.query)
self.ranking = [
self.ranker.next() for _ in range(self.ranker.document_count())
]
@staticmethod
def count_swaps(single_start, list1, list2):
"""
Count the number of pairs that are swapped.
If the lists aren't a swapped version of each other an error is raised
"""
lislen = len(list1)
if lislen != len(list2):
raise ValueError("Lists are not the same size")
if single_start and not list1[0] == list2[0]:
raise ValueError("First element not the same")
if bool(single_start) ^ lislen % 2 and list1[-1] != list2[-1]:
# in this case there is a left over item at the end of the list
raise ValueError("Last element not the same")
swapped = 0
for i in xrange(single_start, lislen - 1, 2):
if list1[i] == list2[i + 1] and list2[i] == list1[i + 1]:
swapped += 1
elif list1[i] != list2[i] or list2[i + 1] != list1[i + 1]:
raise ValueError("Impossible swap")
return swapped
def test_prob_0(self):
"""
Test perturbing with a probability of 0
"""
new_ranked, single_start = ProbabilisticPerturbator(0).perturb(
self.ranker, self.query)
self.assertEqual(new_ranked, self.ranking)
def test_prob_1(self):
"""
Test perturbing with a probability of 1
"""
new_ranked, single_start = ProbabilisticPerturbator(1).perturb(
self.ranker, self.query)
self.assertNotEqual(new_ranked, self.ranking)
swaps = self.count_swaps(single_start, new_ranked, self.ranking)
self.assertEqual(swaps, (len(self.ranking) - single_start) // 2)
@given(floats(0, 1))
def test_prob_hypothesis(self, probability):
perturb = ProbabilisticPerturbator(probability).perturb
self.assertLessEqual(
sum(
self.count_swaps(
*(list(reversed(perturb(self.ranker, self.query))) +
[self.ranking])) for _ in range(self.TRIES)) /
self.TRIES * 2 / len(self.ranking) - probability, self.MARGIN)
class TestPerturbation(unittest.TestCase):
TRIES = 1000
MARGIN = 5 / TRIES
def setUp(self):
# Create ranker to test with
ranker_args = ['3']
ranker_tie = 'random'
init_weights = 'random'
feature_count = 5
self.ranker = DRF(
ranker_args,
ranker_tie,
feature_count,
sample=None,
init=init_weights
)
# Create queries to test with
test_queries = """
1 qid:373 1:0.089908 2:0.531250 3:0.500000 4:0.500000 5:0.156538
0 qid:373 1:0.066055 2:0.171875 3:0.000000 4:0.250000 5:0.084715
0 qid:373 1:0.148624 2:0.015625 3:0.250000 4:0.250000 5:0.151013
0 qid:373 1:0.099083 2:0.250000 3:0.500000 4:0.750000 5:0.134438
0 qid:373 1:0.051376 2:0.078125 3:0.250000 4:0.250000 5:0.060773
0 qid:373 1:0.045872 2:1.000000 3:0.250000 4:0.250000 5:0.163904
"""
query_fh = cStringIO.StringIO(test_queries)
self.query = query.Queries(query_fh, feature_count)['373']
query_fh.close()
# Save the original ranking
self.ranker.init_ranking(self.query)
self.ranking = [
self.ranker.next() for _ in range(self.ranker.document_count())
]
@staticmethod
def count_swaps(single_start, list1, list2):
"""
Count the number of pairs that are swapped.
If the lists aren't a swapped version of each other an error is raised
"""
lislen = len(list1)
if lislen != len(list2):
raise ValueError("Lists are not the same size")
if single_start and not list1[0] == list2[0]:
raise ValueError("First element not the same")
if bool(single_start) ^ lislen % 2 and list1[-1] != list2[-1]:
# in this case there is a left over item at the end of the list
raise ValueError("Last element not the same")
swapped = 0
for i in xrange(single_start, lislen - 1, 2):
if list1[i] == list2[i+1] and list2[i] == list1[i+1]:
swapped += 1
elif list1[i] != list2[i] or list2[i+1] != list1[i+1]:
raise ValueError("Impossible swap")
return swapped
def test_prob_0(self):
"""
Test perturbing with a probability of 0
"""
new_ranked, single_start = ProbabilisticPerturbator(0).perturb(
self.ranker,
self.query
)
self.assertEqual(new_ranked, self.ranking)
def test_prob_1(self):
"""
Test perturbing with a probability of 1
"""
new_ranked, single_start = ProbabilisticPerturbator(1).perturb(
self.ranker,
self.query
)
self.assertNotEqual(new_ranked, self.ranking)
swaps = self.count_swaps(single_start, new_ranked, self.ranking)
self.assertEqual(swaps, (len(self.ranking) - single_start) // 2)
@given(floats(0, 1))
def test_prob_hypothesis(self, probability):
perturb = ProbabilisticPerturbator(probability).perturb
self.assertLessEqual(sum(
self.count_swaps(*(
list(
reversed(perturb(self.ranker, self.query))
) + [self.ranking]
)
)
for _ in range(self.TRIES)
) / self.TRIES * 2 / len(self.ranking) - probability, self.MARGIN)