def testGenerateShardingMapsWithoutStoryTimingData(self):
# Two tests benchmarks are to be sharded between 3 machines. The first one
# has 4 stories, each repeat 2 times. The second one has 4 stories
# without repeat. Without any assumption about timing, the best sharding
# is to shard the first 2 stories of 'foo_benchmark' on shard 1, the next
# two stories of 'foo_benchmark' on shard 2, and 'bar_benchmark' entirely on
# shard 3.
timing_data = []
benchmarks_data = [
FakeBenchmarkConfig(
'foo_benchmark', ['foo_1', 'foo_2', 'foo_3', 'foo_4'], 2),
FakeBenchmarkConfig(
'bar_benchmark', ['bar_1', 'bar_2', 'bar_3', 'bar_4'], 1),
]
sharding_map = sharding_map_generator.generate_sharding_map(
benchmarks_data, timing_data, 3, None)
self.assertEquals(
sharding_map['0']['benchmarks'],
collections.OrderedDict([('bar_benchmark', {'abridged': False})]))
self.assertEquals(
sharding_map['1']['benchmarks'],
collections.OrderedDict([('foo_benchmark',
{'end': 2, 'abridged': False})]))
self.assertEquals(
sharding_map['2']['benchmarks'],
collections.OrderedDict([('foo_benchmark',
{'begin': 2, 'abridged': False})]))
def testGenerateShardingMapsWithoutStoryTimingData(self):
# 3 benchmarks are to be sharded between 3 machines. The first one
# has 4 stories, each repeat 2 times. The second one has 4 stories
# without repeat. Without any assumption about timing, the best sharding
# is to put each benchmark on its own device. Repeats do not necessarily
# imply that a story will take longer than another story that is not
# repeated. This is because short stories tend to be repeated, whereas long
# stories tend to not be repeated.
timing_data = []
benchmarks_data = [
FakeBenchmarkConfig(
'a_benchmark', ['a_1', 'a_2', 'a_3', 'a_4'], 2),
FakeBenchmarkConfig(
'b_benchmark', ['b_1', 'b_2', 'b_3', 'b_4'], 1),
FakeBenchmarkConfig(
'c_benchmark', ['c_1', 'c_2', 'c_3', 'c_4'], 1),
]
sharding_map = sharding_map_generator.generate_sharding_map(
benchmarks_data, timing_data, 3, None)
self.assertEquals(
sharding_map['0']['benchmarks'],
collections.OrderedDict([('a_benchmark', {'abridged': False})]))
self.assertEquals(
sharding_map['1']['benchmarks'],
collections.OrderedDict([('b_benchmark', {'abridged': False})]))
self.assertEquals(
sharding_map['2']['benchmarks'],
collections.OrderedDict([('c_benchmark', {'abridged': False})]))
def testGeneratePerfSharding(self):
test_data_dir = os.path.join(os.path.dirname(__file__), 'test_data')
with open(os.path.join(test_data_dir,
'benchmarks_to_shard.json')) as f:
raw_configs = json.load(f)
benchmarks_to_shard = [
FakeBenchmarkConfig(c['name'], c['stories'], c['repeat'])
for c in raw_configs
]
with open(os.path.join(test_data_dir, 'test_timing_data.json')) as f:
timing_data = json.load(f)
with open(os.path.join(test_data_dir,
'test_timing_data_1_build.json')) as f:
timing_data_single_build = json.load(f)
sharding_map = sharding_map_generator.generate_sharding_map(
benchmarks_to_shard, timing_data, num_shards=5, debug=False)
results = sharding_map_generator.test_sharding_map(
sharding_map, benchmarks_to_shard, timing_data_single_build)
shards_timing = []
for shard in results:
shards_timing.append(results[shard]['full_time'])
self.assertTrue(max(shards_timing) - min(shards_timing) < 600)
def testGenerateAndTestShardingMapWithCrossDeviceTargetCap(self):
benchmarks_data, timing_data, = self._generate_test_data(
[[10, 20, 30], [45, 35, 25, 15], [50, 40, 30, 42, 10]])
target_devices = {'benchmark_2': {'story_0': 5}}
sharding_map = sharding_map_generator.generate_sharding_map(
benchmarks_data, timing_data, 3, None, target_devices)
self.assertIn('benchmark_2', sharding_map['0']['benchmarks'])
self.assertIn('benchmark_2', sharding_map['1']['benchmarks'])
self.assertIn('benchmark_2', sharding_map['2']['benchmarks'])
def testGenerateAndTestShardingMap(self):
benchmarks_data, timing_data, = self._generate_test_data(
[[60, 56, 57], [66, 54, 80, 4], [2, 8, 7, 37, 2]])
timing_data_for_testing = copy.deepcopy(timing_data)
sharding_map = sharding_map_generator.generate_sharding_map(
benchmarks_data, timing_data, 3, None)
results = sharding_map_generator.test_sharding_map(sharding_map,
benchmarks_data, timing_data_for_testing)
self.assertEqual(results['0']['full_time'], 116)
self.assertEqual(results['1']['full_time'], 177)
self.assertEqual(results['2']['full_time'], 140)
def _GenerateShardMap(builder, num_of_shards, output_path, debug):
timing_data = []
if builder:
with open(builder.timing_file_path) as f:
timing_data = json.load(f)
benchmarks_to_shard = list(builder.benchmark_configs)
sharding_map = sharding_map_generator.generate_sharding_map(
benchmarks_to_shard,
timing_data,
num_shards=num_of_shards,
debug=debug)
_DumpJson(sharding_map, output_path)
def GenerateShardMap(builder, num_of_shards, debug=False):
timing_data = []
if builder:
with open(builder.timing_file_path) as f:
timing_data = json.load(f)
benchmarks_to_shard = (list(builder.benchmark_configs) +
list(builder.executables))
repeat_config = cross_device_test_config.TARGET_DEVICES.get(
builder.name, {})
sharding_map = sharding_map_generator.generate_sharding_map(
benchmarks_to_shard,
timing_data,
num_shards=num_of_shards,
debug=debug,
repeat_config=repeat_config)
return sharding_map
def _GenerateShardMap(builder, num_of_shards, output_path, debug):
timing_data = []
if builder:
with open(builder.timing_file_path) as f:
timing_data = json.load(f)
benchmarks_to_shard = (list(builder.benchmark_configs) +
list(builder.executables))
target_devices = cross_device_test_config.TARGET_DEVICES.get(
builder.name, {})
sharding_map = sharding_map_generator.generate_sharding_map(
benchmarks_to_shard,
timing_data,
num_shards=num_of_shards,
debug=debug,
target_devices=target_devices)
_DumpJson(sharding_map, output_path)
def testGenerateAndTestShardingMap(self):
timing_data, all_stories = self._init_sample_timing_data(
[[60, 56, 57], [66, 54, 80, 4], [2, 8, 7, 37, 2]])
timing_data_for_testing = copy.deepcopy(timing_data)
sharding_map = sharding_map_generator.generate_sharding_map(
timing_data, all_stories, 3, None)
fd_map, map_path = tempfile.mkstemp(suffix='.json')
try:
with os.fdopen(fd_map, 'w') as f:
json.dump(sharding_map, f)
results = sharding_map_generator.test_sharding_map(
map_path, timing_data_for_testing, all_stories)
self.assertEqual(results['0']['full_time'], 173)
self.assertEqual(results['1']['full_time'], 120)
self.assertEqual(results['2']['full_time'], 140)
finally:
os.remove(map_path)
def testGenerateAndTestShardingMapWithBenchmarkRepeats(self):
benchmarks_data, timing_data, = self._generate_test_data(
[[10, 20, 30], [45, 35, 25, 15], [50, 40, 30, 20, 10]])
repeat_config = {'benchmark_0': 2, 'benchmark_1': {'story_2': 3}}
sharding_map = sharding_map_generator.generate_sharding_map(
benchmarks_data, timing_data, 3, None, repeat_config)
self.assertIn('benchmark_0', sharding_map['0']['benchmarks'])
# only the 'abridged' key when the whole benchmark is on this shard
self.assertEqual(1,
len(sharding_map['0']['benchmarks']['benchmark_0']))
self.assertIn('benchmark_0', sharding_map['1']['benchmarks'])
self.assertEqual(1,
len(sharding_map['1']['benchmarks']['benchmark_0']))
self.assertNotIn('benchmark_0', sharding_map['2']['benchmarks'])
self.assertIn('benchmark_1', sharding_map['0']['benchmarks'])
self.assertIn('benchmark_1', sharding_map['1']['benchmarks'])
self.assertIn('benchmark_1', sharding_map['2']['benchmarks'])
def testGenerateAndTestShardingMapWithBenchmarkRepeatsCrossShards(self):
benchmarks_data, timing_data, = self._generate_test_data(
[[10, 20, 30], [65, 55, 5, 45], [50, 40, 30, 20, 10]])
repeat_config = {'benchmark_1': {'story_2': 10}, 'benchmark_2': 2}
sharding_map = sharding_map_generator.generate_sharding_map(
benchmarks_data, timing_data, 5, None, repeat_config)
# benchmark_2 takes two shards, and thus will be in 4 shards
self.assertIn('benchmark_2', sharding_map['0']['benchmarks'])
self.assertIn('benchmark_2', sharding_map['1']['benchmarks'])
self.assertIn('benchmark_2', sharding_map['2']['benchmarks'])
self.assertIn('benchmark_2', sharding_map['3']['benchmarks'])
self.assertNotIn('benchmark_2', sharding_map['4']['benchmarks'])
self.assertNotIn('benchmark_0', sharding_map['0']['benchmarks'])
self.assertIn('benchmark_0', sharding_map['1']['benchmarks'])
self.assertIn('benchmark_1', sharding_map['0']['benchmarks'])
self.assertIn('benchmark_1', sharding_map['1']['benchmarks'])
self.assertIn('benchmark_1', sharding_map['2']['benchmarks'])
self.assertIn('benchmark_1', sharding_map['3']['benchmarks'])
self.assertIn('benchmark_1', sharding_map['4']['benchmarks'])
def testGenerateAndTestShardingMapWithCrossDevice(self):
benchmarks_data, timing_data, = self._generate_test_data(
[[10, 20, 30], [45, 35, 25, 15], [50, 40, 30, 20, 10]])
repeat_config = {
'benchmark_0': {
'story_1': 2
},
'benchmark_1': {
'story_2': 3
}
}
sharding_map = sharding_map_generator.generate_sharding_map(
benchmarks_data, timing_data, 3, None, repeat_config)
self.assertIn('benchmark_0', sharding_map['0']['benchmarks'])
self.assertIn('benchmark_0', sharding_map['1']['benchmarks'])
self.assertNotIn('benchmark_0', sharding_map['2']['benchmarks'])
self.assertIn('benchmark_1', sharding_map['0']['benchmarks'])
self.assertIn('benchmark_1', sharding_map['1']['benchmarks'])
self.assertIn('benchmark_1', sharding_map['2']['benchmarks'])
