def do_geth_bench(geth_cmd): print("running geth-evm benchmark...\n{}\n".format(geth_cmd)) geth_cmd = shlex.split(geth_cmd) stdoutlines = [] with subprocess.Popen(geth_cmd, cwd=GETH_EVM_DIR, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1) as p: for line in p.stdout: # b'\n'-separated lines print(line.decode(), end='') stdoutlines.append(line.decode()) # pass bytes as is p.wait() msOpRegex = "evm execution time: ([\d]+.[\d]+)ms" qsOpRegex = "evm execution time: ([\d]+.[\d]+)µs" gasregex = "Gas used:\s+(\d+)" # maybe --benchmark_format=json is better so dont have to parse "36.775k" time_line = stdoutlines[0] gas_line = stdoutlines[-3] time_match = re.search(msOpRegex, time_line) time = None if time_match is None: time_match = re.search(qsOpRegex, time_line) time = durationpy.from_str("{}µs".format(time_match.group(1))) else: time = durationpy.from_str("{}ms".format(time_match.group(1))) gas_match = re.search(gasregex, gas_line) gasused = gas_match.group(1) return {'gas_used': gasused, 'time': time.total_seconds()}
def do_wabt_bench(isolated_bench_dir, wabt_cmd): print("\nrunning wabt benchmark...\n{}\n".format(wabt_cmd)) wabt_cmd = shlex.split(wabt_cmd) stdoutlines = [] with subprocess.Popen(wabt_cmd, cwd=isolated_bench_dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True) as p: for line in p.stdout: # b'\n'-separated lines print(line, end='') stdoutlines.append(line) # pass bytes as is p.wait() parse_time_regex = "parse time: (\d+)us" parse_benchline = stdoutlines[3] parse_time_match = re.search(parse_time_regex, parse_benchline) parse_us_time = durationpy.from_str("{}us".format( parse_time_match.group(1))) exec_time_regex = "wabt_interp\s+(\d+) us" # maybe --benchmark_format=json is better so dont have to parse "36.775k" exec_benchline = stdoutlines[-1] exec_time_match = re.search(exec_time_regex, exec_benchline) exec_us_time = durationpy.from_str("{}us".format(exec_time_match.group(1))) return { 'parse_time': parse_us_time.total_seconds(), 'exec_time': exec_us_time.total_seconds() }
def do_v8_bench(scoutts_cmd, scoutts_working_dir): print("running v8 benchmark...\n{}".format(scoutts_cmd)) scoutts_cmd = shlex.split(scoutts_cmd) stdoutlines = [] with subprocess.Popen(scoutts_cmd, cwd=scoutts_working_dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True) as p: for line in p.stdout: # b'\n'-separated lines print(line, end='') stdoutlines.append(line) # pass bytes as is p.wait() parse_time_regex = "benchmark startup took (\d+) seconds and \d+ nanoseconds \(([\d\w\.\s]+)\)" exec_time_regex = "benchmark execution took (\d+) seconds and \d+ nanoseconds \(([\d\w\.\s]+)\)" parse_time_line = stdoutlines[-2] exec_time_line = stdoutlines[-1] parse_time_match = re.search(parse_time_regex, parse_time_line) parse_time_seconds = durationpy.from_str(parse_time_match.group(1) + "s") parse_time_milliseconds = durationpy.from_str( parse_time_match.group(2).replace(" ", "")) parse_time = parse_time_seconds + parse_time_milliseconds exec_time_match = re.search(exec_time_regex, exec_time_line) exec_time_seconds = durationpy.from_str(exec_time_match.group(1) + "s") exec_time_milliseconds = durationpy.from_str( exec_time_match.group(2).replace(" ", "")) exec_time = exec_time_seconds + exec_time_milliseconds return { 'exec_time': exec_time.total_seconds(), 'parse_time': parse_time.total_seconds() }
def parse_parity_bench_output(stdoutlines): nameRegex = "Benchmarking (\w+): Warming up for" timeRegex = "time:\s+\[[\d\.]+\s+\w+\s+([\d\.]+\s+\w+)" # first match test name # then match time, then append result and wait for next test name bench_tests = [] test_name = "" bench_time = None for line in stdoutlines: matchName = re.search(nameRegex, line) if matchName: test_name = matchName.group(1) bench_time = None matchTime = re.search(timeRegex, line) if matchTime: bench_time = matchTime.group(1) bench_time = bench_time.replace(" ", "") # "1.2129 ms" -> "1.2129ms" if bench_time is not None and test_name != "": bench_time = durationpy.from_str(bench_time) bench_tests.append({ 'name': test_name, 'gas': 0, 'time': bench_time.total_seconds() }) print("parsed test result:", bench_tests[-1]) bench_time = 0 test_name = "" return bench_tests
def do_parity_bench(parity_cmd): print("running parity-evm benchmark...\n{}\n".format(parity_cmd)) parity_cmd = shlex.split(parity_cmd) stdoutlines = [] with subprocess.Popen(parity_cmd, cwd=PARITY_EVM_DIR, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True) as p: for line in p.stdout: # b'\n'-separated lines print(line, end='') stdoutlines.append(line) # pass bytes as is p.wait() timeregex = "code avg run time: ([\d\w\.]+)" gasregex = "gas used: (\d+)" # maybe --benchmark_format=json is better so dont have to parse "36.775k" time_line = stdoutlines[-1] gas_line = stdoutlines[-2] time_match = re.search(timeregex, time_line) time = durationpy.from_str(time_match.group(1)) gas_match = re.search(gasregex, gas_line) gasused = gas_match.group(1) return {'gas_used': gasused, 'time': time.total_seconds()}
def do_scoutcpp_bench(scoutcpp_cmd, yaml_working_dir): print("running scout.cpp benchmark...\n{}".format(scoutcpp_cmd)) scoutcpp_cmd = shlex.split(scoutcpp_cmd) stdoutlines = [] with subprocess.Popen(scoutcpp_cmd, cwd=yaml_working_dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True) as p: for line in p.stdout: # b'\n'-separated lines print(line, end='') stdoutlines.append(line) # pass bytes as is p.wait() time_line = stdoutlines[-1] timeregex = "benchmark took ([\d\.e-]+) seconds." time_match = re.search(timeregex, time_line) time_string = "" if "e-" in time_line: time_string = "{:.8f}s".format(float(time_match.group(1))) else: time_string = time_match.group(1) + "s" time_seconds = durationpy.from_str(time_string) return {'exec_time': time_seconds.total_seconds()}
def parse_go_bench_output(stdoutlines): benchRegex = "Benchmark(Precompiled.*)-Gas=([\d]+)\S+\s+\d+\s+([\d\.]+) ns\/op" #opRegexp = re.compile("Benchmark(Op.*)\S+\s+\d+\s+([\d\.]+) ns\/op") bench_tests = [] for line in stdoutlines: match = re.search(benchRegex, line) if match: (name, gas, nanosecs) = (match.group(1), match.group(2), match.group(3)) bench_time = durationpy.from_str("{}ns".format(nanosecs)) bench_tests.append({ 'name': name, 'gas': gas, 'time': bench_time.total_seconds() }) return bench_tests
def bench_rust_binary(rustdir, input_name, native_exec): print("running rust native {}...\n{}".format(input_name, native_exec)) bench_times = [] for i in range(1, RUST_BENCH_REPEATS): rust_process = subprocess.Popen(native_exec, cwd=rustdir, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True) rust_process.wait(None) stdoutlines = [str(line, 'utf8') for line in rust_process.stdout] print(("").join(stdoutlines), end="") elapsedline = stdoutlines[0] elapsedmatch = re.search("Time elapsed in bench\(\) is: ([\w\.]+)", elapsedline) elapsed_time = durationpy.from_str(elapsedmatch[1]) bench_times.append(elapsed_time.total_seconds()) return bench_times
def do_rust_native(native_bin_path): print("running rust-native benchmark...\n{}".format(native_bin_path)) rust_cmd = shlex.split(native_bin_path) stdoutlines = [] with subprocess.Popen(rust_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True) as p: for line in p.stdout: # b'\n'-separated lines print(line, end='') stdoutlines.append(line) # pass bytes as is p.wait() exec_time_regex = "pairing check time: ([\.\w\d]+)" exec_benchline = stdoutlines[-1] exec_time_match = re.search(exec_time_regex, exec_benchline) exec_us_time = durationpy.from_str(exec_time_match.group(1)) return {'exec_time': exec_us_time.total_seconds()}
def do_evmone_bench(evmone_bench_cmd): evmone_cmd = shlex.split(evmone_bench_cmd) print("running evmone benchmark...\n{}".format(evmone_bench_cmd)) stdoutlines = [] with subprocess.Popen(evmone_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True) as p: for line in p.stdout: # b'\n'-separated lines print(line, end='') stdoutlines.append(line) # pass bytes as is p.wait() json_result = json.loads("".join(stdoutlines[2:])) benchmarks = json_result['benchmarks'] benchmark_results = benchmarks[0] gasused = int(benchmark_results['gas_used']) total_time = str( benchmark_results['real_time']) + benchmark_results['time_unit'] time = durationpy.from_str(total_time) return {'gas_used': gasused, 'time': time.total_seconds()}
def do_evmone_bench(evmone_cmd): print("running evmone benchmark...\n{}\n".format(evmone_cmd)) evmone_cmd = shlex.split(evmone_cmd) stdoutlines = [] with subprocess.Popen(evmone_cmd, cwd=EVMONE_BUILD_DIR, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True) as p: for line in p.stdout: # b'\n'-separated lines print(line, end='') stdoutlines.append(line) # pass bytes as is p.wait() timeregex = "external_evm_code\s+(\d+) us" gasregex = "gas_used=([\d\.\w]+)" # maybe --benchmark_format=json is better so dont have to parse "36.775k" benchline = stdoutlines[-1] time_match = re.search(timeregex, benchline) us_time = durationpy.from_str("{}us".format(time_match.group(1))) gas_match = re.search(gasregex, benchline) gasused = gas_match.group(1) return {'gas_used': gasused, 'time': us_time.total_seconds()}