def testConfigImport(self): p = mock.patch(configs.__name__ + '.FLAGS') self.addCleanup(p.stop) mock_flags = p.start() mock_flags.configure_mock(benchmark_config_file='test_import.yml') config = configs.GetUserConfig() self.assertEqual(config['flags']['num_vms'], 3)
def testConfigOverrideFlag(self): p = mock.patch(configs.__name__ + '.FLAGS') self.addCleanup(p.stop) mock_flags = p.start() config_override = [ 'a.vm_groups.default.vm_count=5', 'a.flags.flag=value'] mock_flags.configure_mock(config_override=config_override, benchmark_config_file=None) config = configs.GetUserConfig() self.assertEqual(config['a']['vm_groups']['default']['vm_count'], 5) self.assertEqual(config['a']['flags']['flag'], 'value')
def GetBenchmarksFromFlags(): """Returns a list of benchmarks to run based on the benchmarks flag. If no benchmarks (or sets) are specified, this will return the standard set. If multiple sets or mixes of sets and benchmarks are specified, this will return the union of all sets and individual benchmarks. """ user_config = configs.GetUserConfig() benchmark_config_list = _GetBenchmarksFromUserConfig(user_config) if benchmark_config_list and not FLAGS['benchmarks'].present: return benchmark_config_list benchmark_names = set() for benchmark in FLAGS.benchmarks: if benchmark in BENCHMARK_SETS: benchmark_names |= set(BENCHMARK_SETS[benchmark][BENCHMARK_LIST]) else: benchmark_names.add(benchmark) # Expand recursive sets expanded = set() did_expansion = True while did_expansion: did_expansion = False for benchmark_name in benchmark_names: if (benchmark_name in BENCHMARK_SETS): did_expansion = True benchmark_names.remove(benchmark_name) if (benchmark_name not in expanded): expanded.add(benchmark_name) benchmark_names |= set( BENCHMARK_SETS[benchmark_name][BENCHMARK_LIST]) break valid_benchmarks = _GetValidBenchmarks() # create a list of module, config tuples to return benchmark_config_list = [] for benchmark_name in benchmark_names: if benchmark_name in valid_benchmarks: benchmark_module = valid_benchmarks[benchmark_name] user_config = user_config.get(benchmark_name, {}) benchmark_config_list.append((benchmark_module, user_config)) else: raise ValueError('Benchmark "%s" not valid on os_type "%s"' % (benchmark_name, FLAGS.os_type)) return benchmark_config_list
def GetBenchmarksFromFlags(): """Returns a list of benchmarks to run based on the benchmarks flag. If no benchmarks (or sets) are specified, this will return the standard set. If multiple sets or mixes of sets and benchmarks are specified, this will return the union of all sets and individual benchmarks. """ user_config = configs.GetUserConfig() benchmark_config_list = _GetBenchmarksFromUserConfig(user_config) if benchmark_config_list and not FLAGS['benchmarks'].present: return benchmark_config_list benchmark_names = set() for benchmark in FLAGS.benchmarks: if benchmark in BENCHMARK_SETS: benchmark_names |= set(BENCHMARK_SETS[benchmark][BENCHMARK_LIST]) else: benchmark_names.add(benchmark) # Expand recursive sets expanded = set() did_expansion = True while did_expansion: did_expansion = False for benchmark_name in benchmark_names: if (benchmark_name in BENCHMARK_SETS): did_expansion = True benchmark_names.remove(benchmark_name) if (benchmark_name not in expanded): expanded.add(benchmark_name) benchmark_names |= set(BENCHMARK_SETS[ benchmark_name][BENCHMARK_LIST]) break valid_benchmarks = _GetValidBenchmarks() # create a list of module, config tuples to return benchmark_config_list = [] for benchmark_name in benchmark_names: benchmark_config = user_config.get(benchmark_name, {}) benchmark_name = benchmark_config.get('name', benchmark_name) benchmark_module = valid_benchmarks.get(benchmark_name) if benchmark_module is None: raise ValueError('Benchmark "%s" not valid on os_type "%s"' % (benchmark_name, FLAGS.os_type)) # We need to remove the 'flag_matrix', 'flag_matrix_defs', 'flag_zip', # 'flag_zip_defs', and 'flag_matrix_filters' keys from the config # dictionary since they aren't actually part of the config spec and will # cause errors if they are left in. flag_matrix_name = benchmark_config.pop( 'flag_matrix', None) flag_matrix_name = FLAGS.flag_matrix or flag_matrix_name flag_matrix = benchmark_config.pop( 'flag_matrix_defs', {}).get(flag_matrix_name, {}) flag_matrix_filter = benchmark_config.pop( 'flag_matrix_filters', {}).get(flag_matrix_name) flag_zip_name = benchmark_config.pop( 'flag_zip', None) flag_zip_name = FLAGS.flag_zip or flag_zip_name flag_zip = benchmark_config.pop( 'flag_zip_defs', {}).get(flag_zip_name, {}) zipped_axes = [] crossed_axes = [] if flag_zip: flag_axes = [] for flag, values in flag_zip.iteritems(): flag_axes.append([{flag: v} for v in values]) _AssertZipAxesHaveSameLength(flag_axes) for flag_config in itertools.izip(*flag_axes): config = _GetConfigForAxis(benchmark_config, flag_config) zipped_axes.append((benchmark_module, config)) crossed_axes.append([benchmark_tuple[1]['flags'] for benchmark_tuple in zipped_axes]) for flag, values in flag_matrix.iteritems(): crossed_axes.append([{flag: v} for v in values]) for flag_config in itertools.product(*crossed_axes): config = _GetConfigForAxis(benchmark_config, flag_config) if (flag_matrix_filter and not eval( flag_matrix_filter, {}, config['flags'])): continue benchmark_config_list.append((benchmark_module, config)) return benchmark_config_list
def GetBenchmarksFromFlags(): """Returns a list of benchmarks to run based on the benchmarks flag. If no benchmarks (or sets) are specified, this will return the standard set. If multiple sets or mixes of sets and benchmarks are specified, this will return the union of all sets and individual benchmarks. Raises: ValueError: when benchmark_name is not valid for os_type supplied """ user_config = configs.GetUserConfig() benchmark_config_list = _GetBenchmarksFromUserConfig(user_config) if benchmark_config_list and not FLAGS['benchmarks'].present: return benchmark_config_list benchmark_queue = collections.deque(FLAGS.benchmarks) benchmark_names = [] benchmark_set = set() while benchmark_queue: benchmark = benchmark_queue.popleft() if benchmark in benchmark_set: continue benchmark_set.add(benchmark) if benchmark in BENCHMARK_SETS: benchmark_queue.extendleft( BENCHMARK_SETS[benchmark][BENCHMARK_LIST]) else: benchmark_names.append(benchmark) valid_benchmarks = _GetValidBenchmarks() # create a list of module, config tuples to return benchmark_config_list = [] for benchmark_name in benchmark_names: benchmark_config = user_config.get(benchmark_name, {}) benchmark_name = benchmark_config.get('name', benchmark_name) benchmark_module = valid_benchmarks.get(benchmark_name) if benchmark_module is None: raise ValueError('Benchmark "%s" not valid on os_type "%s"' % (benchmark_name, FLAGS.os_type)) flag_matrix_name = (FLAGS.flag_matrix or benchmark_config.get('flag_matrix', None)) flag_zip_name = (FLAGS.flag_zip or benchmark_config.get('flag_zip', None)) _AssertFlagMatrixAndZipDefsExist(benchmark_config, flag_matrix_name, flag_zip_name) # We need to remove the 'flag_matrix', 'flag_matrix_defs', 'flag_zip', # 'flag_zip_defs', and 'flag_matrix_filters' keys from the config # dictionary since they aren't actually part of the config spec and will # cause errors if they are left in. benchmark_config.pop('flag_matrix', None) benchmark_config.pop('flag_zip', None) flag_matrix = benchmark_config.pop('flag_matrix_defs', {}).get(flag_matrix_name, {}) flag_matrix_filter = benchmark_config.pop('flag_matrix_filters', {}).get( flag_matrix_name, {}) flag_zip = benchmark_config.pop('flag_zip_defs', {}).get(flag_zip_name, {}) zipped_axes = [] crossed_axes = [] if flag_zip: flag_axes = [] for flag, values in flag_zip.iteritems(): flag_axes.append([{flag: v} for v in values]) _AssertZipAxesHaveSameLength(flag_axes) for flag_config in itertools.izip(*flag_axes): config = _GetConfigForAxis(benchmark_config, flag_config) zipped_axes.append((benchmark_module, config)) crossed_axes.append([ benchmark_tuple[1]['flags'] for benchmark_tuple in zipped_axes ]) for flag, values in sorted(flag_matrix.iteritems()): crossed_axes.append([{flag: v} for v in values]) for flag_config in itertools.product(*crossed_axes): config = _GetConfigForAxis(benchmark_config, flag_config) if (flag_matrix_filter and not eval(flag_matrix_filter, {}, config['flags'])): continue benchmark_config_list.append((benchmark_module, config)) return benchmark_config_list