def _GetConfigForAxis(benchmark_config, flag_config): config = copy.copy(benchmark_config) config_local_flags = config.get('flags', {}) config['flags'] = copy.deepcopy(configs.GetConfigFlags()) config['flags'].update(config_local_flags) for setting in flag_config: config['flags'].update(setting) return config
def GetBenchmarksFromFlags(): """Returns a list of benchmarks to run based on the benchmarks flag. If no benchmarks (or sets) are specified, this will return the standard set. If multiple sets or mixes of sets and benchmarks are specified, this will return the union of all sets and individual benchmarks. """ user_config = configs.GetUserConfig() benchmark_config_list = _GetBenchmarksFromUserConfig(user_config) if benchmark_config_list and not FLAGS['benchmarks'].present: return benchmark_config_list benchmark_names = set() for benchmark in FLAGS.benchmarks: if benchmark in BENCHMARK_SETS: benchmark_names |= set(BENCHMARK_SETS[benchmark][BENCHMARK_LIST]) else: benchmark_names.add(benchmark) # Expand recursive sets expanded = set() did_expansion = True while did_expansion: did_expansion = False for benchmark_name in benchmark_names: if (benchmark_name in BENCHMARK_SETS): did_expansion = True benchmark_names.remove(benchmark_name) if (benchmark_name not in expanded): expanded.add(benchmark_name) benchmark_names |= set( BENCHMARK_SETS[benchmark_name][BENCHMARK_LIST]) break valid_benchmarks = _GetValidBenchmarks() # create a list of module, config tuples to return benchmark_config_list = [] for benchmark_name in benchmark_names: benchmark_config = user_config.get(benchmark_name, {}) benchmark_name = benchmark_config.get('name', benchmark_name) benchmark_module = valid_benchmarks.get(benchmark_name) if benchmark_module is None: raise ValueError('Benchmark "%s" not valid on os_type "%s"' % (benchmark_name, FLAGS.os_type)) # We need to remove the 'flag_matrix', 'flag_matrix_defs', and # 'flag_matrix_filters' keys from the config dictionairy since # they aren't actually part of the config spec and will cause # errors if they are left in. flag_matrix_name = benchmark_config.pop('flag_matrix', None) flag_matrix_name = FLAGS.flag_matrix or flag_matrix_name flag_matrix = benchmark_config.pop('flag_matrix_defs', {}).get(flag_matrix_name, {}) flag_matrix_filter = benchmark_config.pop('flag_matrix_filters', {}).get(flag_matrix_name) flag_axes = [] for flag, values in flag_matrix.iteritems(): flag_axes.append([{flag: v} for v in values]) for flag_config in itertools.product(*flag_axes): config = copy.copy(benchmark_config) config_local_flags = config.get('flags', {}) config['flags'] = copy.deepcopy(configs.GetConfigFlags()) config['flags'].update(config_local_flags) for setting in flag_config: config['flags'].update(setting) if (flag_matrix_filter and not eval(flag_matrix_filter, {}, config['flags'])): continue benchmark_config_list.append((benchmark_module, config)) return benchmark_config_list