def test_multiple_unequal_shard(engine):
buckets = [str(i) for i in range(10)]
entries = [str(i) for i in range(99)]
s = Sharder(engine, 'homedir', buckets)
for e in entries:
s.shard(e)
shards = {}
for e in entries:
shard = s.shard(e)
if shard in shards:
shards[shard] += 1
else:
shards[shard] = 1
assert len(shards) == 10
assert sum(shards.values()) == 99
assert sorted(shards.values()) == [9, 10, 10, 10, 10, 10, 10, 10, 10, 10]
def test_multiple_equal_shard(engine):
buckets = [str(i) for i in range(10)]
entries = [str(i) for i in range(100)]
s = Sharder(engine, 'homedir', buckets)
for e in entries:
s.shard(e)
shards = {}
for e in entries:
shard = s.shard(e)
if shard in shards:
shards[shard] += 1
else:
shards[shard] = 1
assert len(shards) == 10
assert sum(shards.values()) == 100
for shard, count in shards.items():
assert count == 10
def test_multiple_unequal_shards(engine):
"""
Check that when the number of entries isn't divisible by the number of
hubs we still shard as fairly as possible.
"""
hubs = [str(i) for i in range(10)]
entries = [str(i) for i in range(99)]
s = Sharder(engine, 'hub', hubs, log=log.app_log)
[s.shard(e) for e in entries]
shards = {}
for e in entries:
shard = s.shard(e)
if shard in shards:
shards[shard] += 1
else:
shards[shard] = 1
assert len(shards) == 10
assert sum(shards.values()) == 99
assert sorted(shards.values()) == [9, 10, 10, 10, 10, 10, 10, 10, 10, 10]
def test_multiple_equal_shards(engine):
"""
Check that we shard entries equally across hubs. If we have 10 hubs
and 100 entries, each hub should see 10 shards
"""
hubs = [str(i) for i in range(10)]
entries = [str(i) for i in range(100)]
s = Sharder(engine, 'hub', hubs, log=log.app_log)
[s.shard(e) for e in entries]
shards = {}
for e in entries:
shard = s.shard(e)
if shard in shards:
shards[shard] += 1
else:
shards[shard] = 1
assert len(shards) == 10
assert sum(shards.values()) == 100
for shard, count in shards.items():
assert count == 10
def test_single_shard(engine):
s = Sharder(engine, 'hub', ['hub-1'], log=log.app_log)
assert s.shard('user') == 'hub-1'
def test_single_shard(engine):
s = Sharder(engine, 'homedir', ['nfs-a'])
assert s.shard('yuvipanda') == 'nfs-a'
assert s.shard('yuvipanda') == 'nfs-a'