def test_multiple_unequal_shard(engine): buckets = [str(i) for i in range(10)] entries = [str(i) for i in range(99)] s = Sharder(engine, 'homedir', buckets) for e in entries: s.shard(e) shards = {} for e in entries: shard = s.shard(e) if shard in shards: shards[shard] += 1 else: shards[shard] = 1 assert len(shards) == 10 assert sum(shards.values()) == 99 assert sorted(shards.values()) == [9, 10, 10, 10, 10, 10, 10, 10, 10, 10]
def test_multiple_equal_shard(engine): buckets = [str(i) for i in range(10)] entries = [str(i) for i in range(100)] s = Sharder(engine, 'homedir', buckets) for e in entries: s.shard(e) shards = {} for e in entries: shard = s.shard(e) if shard in shards: shards[shard] += 1 else: shards[shard] = 1 assert len(shards) == 10 assert sum(shards.values()) == 100 for shard, count in shards.items(): assert count == 10
def test_multiple_unequal_shards(engine): """ Check that when the number of entries isn't divisible by the number of hubs we still shard as fairly as possible. """ hubs = [str(i) for i in range(10)] entries = [str(i) for i in range(99)] s = Sharder(engine, 'hub', hubs, log=log.app_log) [s.shard(e) for e in entries] shards = {} for e in entries: shard = s.shard(e) if shard in shards: shards[shard] += 1 else: shards[shard] = 1 assert len(shards) == 10 assert sum(shards.values()) == 99 assert sorted(shards.values()) == [9, 10, 10, 10, 10, 10, 10, 10, 10, 10]
def test_multiple_equal_shards(engine): """ Check that we shard entries equally across hubs. If we have 10 hubs and 100 entries, each hub should see 10 shards """ hubs = [str(i) for i in range(10)] entries = [str(i) for i in range(100)] s = Sharder(engine, 'hub', hubs, log=log.app_log) [s.shard(e) for e in entries] shards = {} for e in entries: shard = s.shard(e) if shard in shards: shards[shard] += 1 else: shards[shard] = 1 assert len(shards) == 10 assert sum(shards.values()) == 100 for shard, count in shards.items(): assert count == 10
def test_single_shard(engine): s = Sharder(engine, 'hub', ['hub-1'], log=log.app_log) assert s.shard('user') == 'hub-1'
def test_single_shard(engine): s = Sharder(engine, 'homedir', ['nfs-a']) assert s.shard('yuvipanda') == 'nfs-a' assert s.shard('yuvipanda') == 'nfs-a'