def test__unbiasedness_constraints(self):
lists = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
b = il.Optimized(lists, sample_num=27)
res = b._unbiasedness_constraints(lists, b._rankings)
assert res.shape[0] == (3-1)*3
assert res.shape[1] == len(b._rankings)
lists = [[1, 2], [2, 3]]
b = il.Optimized(lists, sample_num=3)
res = b._unbiasedness_constraints(lists, b._rankings)
ideal = {
(1, 2): [1-1.0/3, 1.5-1.0-1.0/3],
(2, 3): [0.5-1.0, 0.5+1.0/3-1.5],
(2, 1): [0.5-1.0, 1.5-1.0-1.0/3]}
assert res.shape == (2, 3)
for i in range(2):
for j, r in enumerate(b._rankings):
res[i, j] = ideal[tuple(r)][i]
lists = [[1, 2], [1, 3], [1, 4]]
b = il.Optimized(lists, sample_num=3)
rankings = []
r = CreditRanking(num_rankers=len(lists), contents=[1, 2])
r.credits = {
0: {1: 1.0, 2: 0.5}, 1: {1: 1.0, 2: 1.0/3}, 2: {1: 1.0, 2: 1.0/3}}
rankings.append(r)
r = CreditRanking(num_rankers=len(lists), contents=[1, 3])
r.credits = {
0: {1: 1.0, 3: 1.0/3}, 1: {1: 1.0, 3: 0.5}, 2: {1: 1.0, 3: 1.0/3}}
rankings.append(r)
r = CreditRanking(num_rankers=len(lists), contents=[1, 4])
r.credits = {
0: {1: 1.0, 4: 1.0/3}, 1: {1: 1.0, 4: 1.0/3}, 2: {1: 1.0, 4: 0.5}}
rankings.append(r)
res = b._unbiasedness_constraints(lists, rankings)
assert res.shape == (4, 3)
assert res[0, 0] == 0 # Rank 1, Ranking 1, System 1 - System 2
assert res[0, 1] == 0 # Rank 1, Ranking 2, System 1 - System 2
assert res[0, 2] == 0 # Rank 1, Ranking 3, System 1 - System 2
# Rank 2, Ranking 1, System 1 - System 2
self.assert_almost_equal(res[1, 0], 0.5-1.0/3)
# Rank 2, Ranking 2, System 1 - System 2
self.assert_almost_equal(res[1, 1], 1.0/3-0.5)
assert res[1, 2] == 0 # Rank 2, Ranking 3, System 1 - System 2
assert res[2, 0] == 0 # Rank 1, Ranking 1, System 2 - System 3
assert res[2, 1] == 0 # Rank 1, Ranking 2, System 2 - System 3
assert res[2, 2] == 0 # Rank 1, Ranking 3, System 2 - System 3
assert res[3, 0] == 0 # Rank 2, Ranking 1, System 2 - System 3
# Rank 2, Ranking 2, System 2 - System 3
self.assert_almost_equal(res[3, 1], 0.5-1.0/3)
# Rank 2, Ranking 3, System 2 - System 3
self.assert_almost_equal(res[3, 2], 1.0/3-0.5)
def test_secure_sampling(self):
b = il.Optimized([[1, 2], [2, 3]],
sample_num=1000,
secure_sampling=True)
b._sample_rankings()
assert [[1, 2], [2, 1], [2, 3]] == sorted(b._rankings)
assert [1.0 / 3] * 3 == list(b._probabilities)
def test_evaluate(self):
lists = [[1, 2], [2, 3]]
b = il.Optimized(lists, sample_num=3)
samples = [b.interleave() for i in range(1000)]
rankings = {tuple(r): r for r in samples}
ideals = {
(1, 2): {
0: [(0, 1)],
1: [(0, 1)],
2: [(1, 0)],
3: []
},
(2, 1): {
0: [(0, 1)],
1: [(1, 0)],
2: [(0, 1)],
3: []
},
(2, 3): {
0: [(1, 0)],
1: [(1, 0)],
2: [(1, 0)],
3: []
},
}
for r in rankings:
self.evaluate(il.Optimized, rankings[r], [0, 1], set(ideals[r][0]))
self.evaluate(il.Optimized, rankings[r], [0], set(ideals[r][1]))
self.evaluate(il.Optimized, rankings[r], [1], set(ideals[r][2]))
self.evaluate(il.Optimized, rankings[r], [], set(ideals[r][3]))
def test_interleave(self):
lists = [[1, 2], [2, 3]]
b = il.Optimized(lists, sample_num=3)
rankings, probabilities = zip(*b.ranking_distribution)
assert set([(1, 2), (2, 1),
(2, 3)]) == set([tuple(r) for r in rankings])
ideals = {
(1, 2): 0.4285714273469387,
(2, 1): 0.37142857025306114,
(2, 3): 0.20000000240000002
}
for i in range(len(probabilities)):
r = rankings[i]
self.assert_almost_equal(probabilities[i],
ideals[tuple(r)],
error_rate=0.01)
trials = 200000
counts = {(1, 2): 0, (2, 1): 0, (2, 3): 0}
for i in range(trials):
r = b.interleave()
counts[tuple(r)] += 1
for r, c in counts.items():
self.assert_almost_equal(float(c) / trials,
ideals[tuple(r)],
error_rate=0.01)
def test__compute_probabilities(self):
lists = [[1, 2], [2, 3]]
b = il.Optimized(lists, sample_num=3)
rankings = []
r = CreditRanking(num_rankers=len(lists), contents=[1, 2])
r.credits = {0: {1: 1.0, 2: 0.5}, 1: {1: 1.0 / 3, 2: 1.0}}
rankings.append(r)
r = CreditRanking(num_rankers=len(lists), contents=[2, 1])
r.credits = {0: {1: 1.0, 2: 0.5}, 1: {1: 1.0 / 3, 2: 1.0}}
rankings.append(r)
r = CreditRanking(num_rankers=len(lists), contents=[2, 3])
r.credits = {0: {2: 0.5, 3: 1.0 / 3}, 1: {2: 1.0, 3: 0.5}}
rankings.append(r)
is_success, p, minimum = b._compute_probabilities(lists, rankings)
assert is_success
assert (p >= 0).all()
assert (p <= 1).all()
assert minimum >= 0
self.assert_almost_equal(np.sum(p), 1)
self.assert_almost_equal(np.inner([1 - 1.0 / 3, -0.5, -0.5], p), 0)
self.assert_almost_equal(
np.inner([0.5 - 1.0 / 3, 0.5 - 1.0 / 3, -1 + 1.0 / 3], p), 0)
self.assert_almost_equal(p[0], 0.4285714273469387)
self.assert_almost_equal(p[1], 0.37142857025306114)
self.assert_almost_equal(p[2], 0.20000000240000002)
def test_init_sampling(self):
b = il.Optimized([[1, 2], [2, 3]], sample_num=3)
samples = set([tuple(b.interleave()) for i in range(1000)])
assert len(samples) == 3
assert (1, 2) in samples
assert (2, 1) in samples
assert (2, 3) in samples
def test__sensitivity(self):
lists = [[1, 2], [2, 3]]
b = il.Optimized(lists, sample_num=3)
rankings = []
r = CreditRanking(num_rankers=len(lists), contents=[1, 2])
r.credits = {0: {1: 1.0, 2: 0.5}, 1: {1: 1.0/3, 2: 1.0}}
rankings.append(r)
r = CreditRanking(num_rankers=len(lists), contents=[2, 1])
r.credits = {0: {1: 1.0, 2: 0.5}, 1: {1: 1.0/3, 2: 1.0}}
rankings.append(r)
r = CreditRanking(num_rankers=len(lists), contents=[2, 3])
r.credits = {0: {2: 0.5, 3: 1.0/3}, 1: {2: 1.0, 3: 0.5}}
rankings.append(r)
res = b._sensitivity(lists, rankings)
assert len(res) == 3
assert res[0] == ((1 + 0.5 * 0.5 - (1.75+1.0/3)/2) ** 2 + (1.0/3+0.5 - (1.75+1.0/3)/2) ** 2)
assert res[1] == ((0.5 + 0.5 - (2.0+0.5/3)/2) ** 2 + (1.0+1.0/3*0.5 - (2.0+0.5/3)/2) ** 2)
assert res[2] == ((0.5 + 0.5 * 1.0/3 - (1.75+0.5/3)/2) ** 2\
+ (1.0 + 0.5 * 0.5 - (1.75+0.5/3)/2) ** 2)
def test_dump(self, tmpdir):
tmpfile = str(tmpdir) + '/optimized.json'
b = il.Optimized([[1, 2], [2, 3]], sample_num=3)
b.dump_rankings(tmpfile)
with open(tmpfile, 'r') as f:
obj = json.load(f)
# Test keys
s = {str(hash(r)) for r in b._rankings}
assert s == set(obj.keys())
# Test rankings
l1 = sorted(b._rankings)
l2 = sorted([v['ranking']['ranking_list'] for v in obj.values()])
assert l1 == l2
# Test credits
l1 = [r.credits for r in b._rankings]
l2 = [v['ranking']['credits'] for v in obj.values()]
f = lambda d: {int(k): v for k, v in d.items()}
for index, item in enumerate(l2):
l2[index] = {int(k): f(v) for k, v in item.items()}
g = lambda d: sorted((k, sorted(v.items())) for k, v in d.items())
assert sorted(g(i1) for i1 in l1) == sorted(g(i2) for i2 in l2)
def test_raise_value_error(self):
with pytest.raises(ValueError):
# None for `sample_num` is not allowed
il.Optimized([[0], [1]])
