class HistDocumentConstraints(AbstractHistInterleavedComparison):
"""Document constraints method, applied to historical data."""
def __init__(self, arg_str=None):
if arg_str:
self.dc = DocumentConstraints(arg_str)
else:
self.dc = DocumentConstraints()
def infer_outcome(self, l, a, c, target_r1, target_r2, query):
"""count clicks within the top-k interleaved list"""
c = asarray(c)
click_ids = where(c == 1)[0]
if not len(click_ids): # no clicks, will be a tie
return 0
# get ranked list for each ranker
target_r1.init_ranking(query)
target_r2.init_ranking(query)
length = min(target_r1.document_count(), target_r2.document_count(),
len(l))
a = ([], [])
for _ in range(length):
a[0].append(target_r1.next())
a[1].append(target_r2.next())
a = (asarray(a[0]), asarray(a[1]))
# check for violated constraints
c1, c2 = self.dc.check_constraints(l, a, click_ids)
# now we have constraints, not clicks, reverse outcome
return 1 if c1 > c2 else -1 if c2 > c1 else 0
class HistDocumentConstraints(AbstractHistInterleavedComparison):
"""Document constraints method, applied to historical data."""
def __init__(self, arg_str=None):
if arg_str:
self.dc = DocumentConstraints(arg_str)
else:
self.dc = DocumentConstraints()
def infer_outcome(self, l, a, c, target_r1, target_r2, query):
"""count clicks within the top-k interleaved list"""
c = asarray(c)
click_ids = where(c == 1)[0]
if not len(click_ids): # no clicks, will be a tie
return 0
# get ranked list for each ranker
target_r1.init_ranking(query)
target_r2.init_ranking(query)
length = min(target_r1.document_count(), target_r2.document_count(),
len(l))
a = ([], [])
for _ in range(length):
a[0].append(target_r1.next())
a[1].append(target_r2.next())
a = (asarray(a[0]), asarray(a[1]))
# check for violated constraints
c1, c2 = self.dc.check_constraints(l, a, click_ids)
# now we have constraints, not clicks, reverse outcome
return 1 if c1 > c2 else -1 if c2 > c1 else 0
def __init__(self, arg_str=None):
if arg_str:
self.dc = DocumentConstraints(arg_str)
else:
self.dc = DocumentConstraints()
def testDocumentConstraints(self):
dc = DocumentConstraints()
r1 = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
(interleaved_list, assignments) = dc.interleave(r1, r2, self.query, 10)
self.assertIn(interleaved_list.tolist(), [[0, 1, 2, 3], [1, 0, 2, 3],
[0, 1, 3, 2], [1, 0, 3, 2]])
self.assertIn(assignments[0].tolist(), [[1, 2, 3, 0], [1, 3, 2, 0]])
self.assertIn(assignments[1].tolist(), [[0, 1, 2, 3], [0, 1, 3, 2]])
o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[1, 0, 0, 0], self.query)
self.assertEqual(o, -1, "l1 should win (1), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[0, 0, 0, 1], self.query)
self.assertEqual(o, -1, "l1 should win (2), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[0, 1, 0, 0], self.query)
self.assertEqual(o, 1, "l2 should win (1), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 2, 3], [0, 1, 3, 2]],
[0, 1, 0, 0], self.query)
# constraints: 0 > 1, 0 > 3
self.assertEqual(o, 1, "l2 should win (2), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 2, 0, 3], [1, 0, 2, 3]],
[0, 1, 1, 0], self.query)
# constraints: 0 > 1, 3 > 1, 0 > 2, 3 > 2
self.assertEqual(o, 1, "l2 should win (3), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[0, 0, 0, 0], self.query)
self.assertEqual(o, 0, "No winner when there are no clicks o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[1, 1, 1, 1], self.query)
self.assertEqual(o, 0, "No winner when all are clicked o = %g" % o)
dc = DocumentConstraints("--constraints 1")
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 2, 3], [3, 0, 1, 2]],
[0, 1, 0, 0], self.query)
# constraint: 0 > 1
self.assertEqual(o, 1, "l2 should win with one constraint, o = %g" % o)
dc = DocumentConstraints("--constraints 2")
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 2, 3], [3, 0, 1, 2]],
[0, 1, 0, 0], self.query)
self.assertEqual(o, 0, "Tie with two constraint types (1), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 2, 3], [1, 2, 0, 3]],
[0, 1, 1, 0], self.query)
# constraints: 0 > 1, 3 > 1, 3 > 2
self.assertEqual(o, 0, "Tie with two constraint types (2), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 4, 3], [1, 0, 3, 2]],
[0, 1, 1, 0], self.query)
self.assertEqual(o, 0, "Tie with two constraint types (3), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 4, 3], [1, 0, 2, 3]],
[0, 1, 1, 0], self.query)
self.assertEqual(o, -1, "l1 should win with two constr., o = %g" % o)
def __init__(self, arg_str=None):
if arg_str:
self.dc = DocumentConstraints(arg_str)
else:
self.dc = DocumentConstraints()
def testDocumentConstraints(self):
dc = DocumentConstraints()
r1 = DeterministicRankingFunction(None, self.weights_1)
r2 = DeterministicRankingFunction(None, self.weights_2)
(interleaved_list, assignments) = dc.interleave(r1, r2, self.query, 10)
self.assertIn(interleaved_list.tolist(),
[[0, 1, 2, 3], [1, 0, 2, 3], [0, 1, 3, 2], [1, 0, 3, 2]])
self.assertIn(assignments[0].tolist(), [[1, 2, 3, 0], [1, 3, 2, 0]])
self.assertIn(assignments[1].tolist(), [[0, 1, 2, 3], [0, 1, 3, 2]])
o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[1, 0, 0, 0], self.query)
self.assertEqual(o, -1, "l1 should win (1), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[0, 0, 0, 1], self.query)
self.assertEqual(o, -1, "l1 should win (2), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[0, 1, 0, 0], self.query)
self.assertEqual(o, 1, "l2 should win (1), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 2, 3], [0, 1, 3, 2]],
[0, 1, 0, 0], self.query)
# constraints: 0 > 1, 0 > 3
self.assertEqual(o, 1, "l2 should win (2), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 2, 0, 3], [1, 0, 2, 3]],
[0, 1, 1, 0], self.query)
# constraints: 0 > 1, 3 > 1, 0 > 2, 3 > 2
self.assertEqual(o, 1, "l2 should win (3), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[0, 0, 0, 0], self.query)
self.assertEqual(o, 0, "No winner when there are no clicks o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 3, 2, 0], [0, 1, 3, 2]],
[1, 1, 1, 1], self.query)
self.assertEqual(o, 0, "No winner when all are clicked o = %g" % o)
dc = DocumentConstraints("--constraints 1")
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 2, 3], [3, 0, 1, 2]],
[0, 1, 0, 0], self.query)
# constraint: 0 > 1
self.assertEqual(o, 1, "l2 should win with one constraint, o = %g" % o)
dc = DocumentConstraints("--constraints 2")
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 2, 3], [3, 0, 1, 2]],
[0, 1, 0, 0], self.query)
self.assertEqual(o, 0, "Tie with two constraint types (1), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 2, 3], [1, 2, 0, 3]],
[0, 1, 1, 0], self.query)
# constraints: 0 > 1, 3 > 1, 3 > 2
self.assertEqual(o, 0, "Tie with two constraint types (2), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 4, 3], [1, 0, 3, 2]],
[0, 1, 1, 0], self.query)
self.assertEqual(o, 0, "Tie with two constraint types (3), o = %g" % o)
o = dc.infer_outcome([1, 0, 3, 2], [[1, 0, 4, 3], [1, 0, 2, 3]],
[0, 1, 1, 0], self.query)
self.assertEqual(o, -1, "l1 should win with two constr., o = %g" % o)
