def execute_benchmark(benchmark, isolated_out_dir, args, rest_args, is_reference, stories=None): start = time.time() # While we are between chartjson and histogram set we need # to determine which output format to look for or see if it was # already passed in in which case that format applies to all benchmarks # in this run. is_histograms = append_output_format(args, rest_args) # Insert benchmark name as first argument to run_benchmark call # which is the first argument in the rest_args. Also need to append # output format and smoke test mode. per_benchmark_args = (rest_args[:1] + [benchmark] + rest_args[1:]) benchmark_name = benchmark if is_reference: # Need to parse out the browser to replace browser flag with # reference build so we run it reference build as well browser_index = 0 for arg in per_benchmark_args: if "browser" in arg: break browser_index = browser_index + 1 per_benchmark_args[browser_index] = '--browser=reference' # Now we need to add in the rest of the reference build args per_benchmark_args.append('--max-failures=5') per_benchmark_args.append('--output-trace-tag=_ref') benchmark_name = benchmark + '.reference' # If we are only running a subset of stories, add in the begin and end # index. if stories: if 'begin' in stories.keys(): per_benchmark_args.append( ('--experimental-story-shard-begin-index=%d' % stories['begin'])) if 'end' in stories.keys(): per_benchmark_args.append( ('--experimental-story-shard-end-index=%d' % stories['end'])) # We don't care exactly what these are. In particular, the perf results # could be any format (chartjson, legacy, histogram). We just pass these # through, and expose these as results for this task. rc, perf_results, json_test_results, benchmark_log = ( run_telemetry_benchmark_as_googletest.run_benchmark( args, per_benchmark_args, is_histograms)) write_results(benchmark_name, perf_results, json_test_results, benchmark_log, isolated_out_dir, False) print_duration('executing benchmark %s' % benchmark_name, start) return rc
def main(): parser = argparse.ArgumentParser() parser.add_argument('--isolated-script-test-output', required=True) parser.add_argument('--isolated-script-test-chartjson-output', required=False) parser.add_argument('--isolated-script-test-perf-output', required=False) parser.add_argument('--isolated-script-test-filter', type=str, required=False) parser.add_argument('--xvfb', help='Start xvfb.', action='store_true') parser.add_argument('--output-format', action='append') parser.add_argument('--builder', required=True) parser.add_argument( '--bot', required=True, help='Bot ID to use to determine which tests to run. Will' ' use //tools/perf/core/benchmark_sharding_map.json' ' with this as a key to determine which benchmarks' ' to execute') args, rest_args = parser.parse_known_args() for output_format in args.output_format: rest_args.append('--output-format=' + output_format) isolated_out_dir = os.path.dirname(args.isolated_script_test_output) with open(sharding_map_path()) as f: sharding_map = json.load(f) sharding = sharding_map[args.builder][args.bot]['benchmarks'] return_code = 0 for benchmark in sharding: # Insert benchmark name as first argument to run_benchmark call per_benchmark_args = rest_args[:1] + [benchmark] + rest_args[1:] # We don't care exactly what these are. In particular, the perf results # could be any format (chartjson, legacy, histogram). We just pass these # through, and expose these as results for this task. rc, perf_results, json_test_results = ( run_telemetry_benchmark_as_googletest.run_benchmark( args, per_benchmark_args)) return_code = return_code or rc benchmark_path = os.path.join(isolated_out_dir, benchmark) os.makedirs(benchmark_path) with open(os.path.join(benchmark_path, 'perf_results.json'), 'w') as f: json.dump(perf_results, f) with open(os.path.join(benchmark_path, 'test_results.json'), 'w') as f: json.dump(json_test_results, f) return return_code
def execute_benchmark(benchmark, isolated_out_dir, args, rest_args, is_reference): # While we are between chartjson and histogram set we need # to determine which output format to look for. # We need to append this both to the args and the per benchmark # args so the run_benchmark call knows what format it is # as well as triggers the benchmark correctly. output_format = None is_histograms = False if benchmark in BENCHMARKS_TO_OUTPUT_HISTOGRAMS: output_format = '--output-format=histograms' is_histograms = True else: output_format = '--output-format=chartjson' # Need to run the benchmark twice on browser and reference build # Insert benchmark name as first argument to run_benchmark call # Need to append output format. per_benchmark_args = (rest_args[:1] + [benchmark] + rest_args[1:] + [output_format]) benchmark_path = None if is_reference: # Need to parse out the browser to replace browser flag with # reference build so we run it reference build as well browser_index = 0 for arg in per_benchmark_args: if "browser" in arg: break browser_index = browser_index + 1 per_benchmark_args[browser_index] = '--browser=reference' # Now we need to add in the rest of the reference build args per_benchmark_args.append('--max-failures=5') per_benchmark_args.append('--output-trace-tag=_ref') benchmark_path = os.path.join(isolated_out_dir, benchmark + '.reference') else: benchmark_path = os.path.join(isolated_out_dir, benchmark) # We don't care exactly what these are. In particular, the perf results # could be any format (chartjson, legacy, histogram). We just pass these # through, and expose these as results for this task. rc, perf_results, json_test_results = ( run_telemetry_benchmark_as_googletest.run_benchmark( args, per_benchmark_args, is_histograms)) os.makedirs(benchmark_path) with open(os.path.join(benchmark_path, 'perf_results.json'), 'w') as f: json.dump(perf_results, f) with open(os.path.join(benchmark_path, 'test_results.json'), 'w') as f: json.dump(json_test_results, f) return rc
def execute_benchmark(benchmark, isolated_out_dir, args, rest_args, is_reference): # While we are between chartjson and histogram set we need # to determine which output format to look for or see if it was # already passed in in which case that format applies to all benchmarks # in this run. is_histograms = append_output_format(benchmark, args, rest_args) # Insert benchmark name as first argument to run_benchmark call # which is the first argument in the rest_args. Also need to append # output format. per_benchmark_args = (rest_args[:1] + [benchmark] + rest_args[1:]) benchmark_name = benchmark if is_reference: # Need to parse out the browser to replace browser flag with # reference build so we run it reference build as well browser_index = 0 for arg in per_benchmark_args: if "browser" in arg: break browser_index = browser_index + 1 per_benchmark_args[browser_index] = '--browser=reference' # Now we need to add in the rest of the reference build args per_benchmark_args.append('--max-failures=5') per_benchmark_args.append('--output-trace-tag=_ref') benchmark_name = benchmark + '.reference' # We don't care exactly what these are. In particular, the perf results # could be any format (chartjson, legacy, histogram). We just pass these # through, and expose these as results for this task. rc, perf_results, json_test_results = ( run_telemetry_benchmark_as_googletest.run_benchmark( args, per_benchmark_args, is_histograms)) write_results(benchmark_name, perf_results, json_test_results, isolated_out_dir, False) return rc