def do_geth_bench(geth_cmd):
print("running geth-evm benchmark...\n{}\n".format(geth_cmd))
geth_cmd = shlex.split(geth_cmd)
stdoutlines = []
with subprocess.Popen(geth_cmd,
cwd=GETH_EVM_DIR,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
bufsize=1) as p:
for line in p.stdout: # b'\n'-separated lines
print(line.decode(), end='')
stdoutlines.append(line.decode()) # pass bytes as is
p.wait()
msOpRegex = "evm execution time: ([\d]+.[\d]+)ms"
qsOpRegex = "evm execution time: ([\d]+.[\d]+)µs"
gasregex = "Gas used:\s+(\d+)"
# maybe --benchmark_format=json is better so dont have to parse "36.775k"
time_line = stdoutlines[0]
gas_line = stdoutlines[-3]
time_match = re.search(msOpRegex, time_line)
time = None
if time_match is None:
time_match = re.search(qsOpRegex, time_line)
time = durationpy.from_str("{}µs".format(time_match.group(1)))
else:
time = durationpy.from_str("{}ms".format(time_match.group(1)))
gas_match = re.search(gasregex, gas_line)
gasused = gas_match.group(1)
return {'gas_used': gasused, 'time': time.total_seconds()}
def do_wabt_bench(isolated_bench_dir, wabt_cmd):
print("\nrunning wabt benchmark...\n{}\n".format(wabt_cmd))
wabt_cmd = shlex.split(wabt_cmd)
stdoutlines = []
with subprocess.Popen(wabt_cmd,
cwd=isolated_bench_dir,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
bufsize=1,
universal_newlines=True) as p:
for line in p.stdout: # b'\n'-separated lines
print(line, end='')
stdoutlines.append(line) # pass bytes as is
p.wait()
parse_time_regex = "parse time: (\d+)us"
parse_benchline = stdoutlines[3]
parse_time_match = re.search(parse_time_regex, parse_benchline)
parse_us_time = durationpy.from_str("{}us".format(
parse_time_match.group(1)))
exec_time_regex = "wabt_interp\s+(\d+) us"
# maybe --benchmark_format=json is better so dont have to parse "36.775k"
exec_benchline = stdoutlines[-1]
exec_time_match = re.search(exec_time_regex, exec_benchline)
exec_us_time = durationpy.from_str("{}us".format(exec_time_match.group(1)))
return {
'parse_time': parse_us_time.total_seconds(),
'exec_time': exec_us_time.total_seconds()
}
def do_v8_bench(scoutts_cmd, scoutts_working_dir):
print("running v8 benchmark...\n{}".format(scoutts_cmd))
scoutts_cmd = shlex.split(scoutts_cmd)
stdoutlines = []
with subprocess.Popen(scoutts_cmd,
cwd=scoutts_working_dir,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
bufsize=1,
universal_newlines=True) as p:
for line in p.stdout: # b'\n'-separated lines
print(line, end='')
stdoutlines.append(line) # pass bytes as is
p.wait()
parse_time_regex = "benchmark startup took (\d+) seconds and \d+ nanoseconds \(([\d\w\.\s]+)\)"
exec_time_regex = "benchmark execution took (\d+) seconds and \d+ nanoseconds \(([\d\w\.\s]+)\)"
parse_time_line = stdoutlines[-2]
exec_time_line = stdoutlines[-1]
parse_time_match = re.search(parse_time_regex, parse_time_line)
parse_time_seconds = durationpy.from_str(parse_time_match.group(1) + "s")
parse_time_milliseconds = durationpy.from_str(
parse_time_match.group(2).replace(" ", ""))
parse_time = parse_time_seconds + parse_time_milliseconds
exec_time_match = re.search(exec_time_regex, exec_time_line)
exec_time_seconds = durationpy.from_str(exec_time_match.group(1) + "s")
exec_time_milliseconds = durationpy.from_str(
exec_time_match.group(2).replace(" ", ""))
exec_time = exec_time_seconds + exec_time_milliseconds
return {
'exec_time': exec_time.total_seconds(),
'parse_time': parse_time.total_seconds()
}
def parse_parity_bench_output(stdoutlines):
nameRegex = "Benchmarking (\w+): Warming up for"
timeRegex = "time:\s+\[[\d\.]+\s+\w+\s+([\d\.]+\s+\w+)"
# first match test name
# then match time, then append result and wait for next test name
bench_tests = []
test_name = ""
bench_time = None
for line in stdoutlines:
matchName = re.search(nameRegex, line)
if matchName:
test_name = matchName.group(1)
bench_time = None
matchTime = re.search(timeRegex, line)
if matchTime:
bench_time = matchTime.group(1)
bench_time = bench_time.replace(" ",
"") # "1.2129 ms" -> "1.2129ms"
if bench_time is not None and test_name != "":
bench_time = durationpy.from_str(bench_time)
bench_tests.append({
'name': test_name,
'gas': 0,
'time': bench_time.total_seconds()
})
print("parsed test result:", bench_tests[-1])
bench_time = 0
test_name = ""
return bench_tests
def do_parity_bench(parity_cmd):
print("running parity-evm benchmark...\n{}\n".format(parity_cmd))
parity_cmd = shlex.split(parity_cmd)
stdoutlines = []
with subprocess.Popen(parity_cmd,
cwd=PARITY_EVM_DIR,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
bufsize=1,
universal_newlines=True) as p:
for line in p.stdout: # b'\n'-separated lines
print(line, end='')
stdoutlines.append(line) # pass bytes as is
p.wait()
timeregex = "code avg run time: ([\d\w\.]+)"
gasregex = "gas used: (\d+)"
# maybe --benchmark_format=json is better so dont have to parse "36.775k"
time_line = stdoutlines[-1]
gas_line = stdoutlines[-2]
time_match = re.search(timeregex, time_line)
time = durationpy.from_str(time_match.group(1))
gas_match = re.search(gasregex, gas_line)
gasused = gas_match.group(1)
return {'gas_used': gasused, 'time': time.total_seconds()}
def do_scoutcpp_bench(scoutcpp_cmd, yaml_working_dir):
print("running scout.cpp benchmark...\n{}".format(scoutcpp_cmd))
scoutcpp_cmd = shlex.split(scoutcpp_cmd)
stdoutlines = []
with subprocess.Popen(scoutcpp_cmd,
cwd=yaml_working_dir,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
bufsize=1,
universal_newlines=True) as p:
for line in p.stdout: # b'\n'-separated lines
print(line, end='')
stdoutlines.append(line) # pass bytes as is
p.wait()
time_line = stdoutlines[-1]
timeregex = "benchmark took ([\d\.e-]+) seconds."
time_match = re.search(timeregex, time_line)
time_string = ""
if "e-" in time_line:
time_string = "{:.8f}s".format(float(time_match.group(1)))
else:
time_string = time_match.group(1) + "s"
time_seconds = durationpy.from_str(time_string)
return {'exec_time': time_seconds.total_seconds()}
def parse_go_bench_output(stdoutlines):
benchRegex = "Benchmark(Precompiled.*)-Gas=([\d]+)\S+\s+\d+\s+([\d\.]+) ns\/op"
#opRegexp = re.compile("Benchmark(Op.*)\S+\s+\d+\s+([\d\.]+) ns\/op")
bench_tests = []
for line in stdoutlines:
match = re.search(benchRegex, line)
if match:
(name, gas, nanosecs) = (match.group(1), match.group(2),
match.group(3))
bench_time = durationpy.from_str("{}ns".format(nanosecs))
bench_tests.append({
'name': name,
'gas': gas,
'time': bench_time.total_seconds()
})
return bench_tests
def bench_rust_binary(rustdir, input_name, native_exec):
print("running rust native {}...\n{}".format(input_name, native_exec))
bench_times = []
for i in range(1, RUST_BENCH_REPEATS):
rust_process = subprocess.Popen(native_exec,
cwd=rustdir,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
shell=True)
rust_process.wait(None)
stdoutlines = [str(line, 'utf8') for line in rust_process.stdout]
print(("").join(stdoutlines), end="")
elapsedline = stdoutlines[0]
elapsedmatch = re.search("Time elapsed in bench\(\) is: ([\w\.]+)",
elapsedline)
elapsed_time = durationpy.from_str(elapsedmatch[1])
bench_times.append(elapsed_time.total_seconds())
return bench_times
def do_rust_native(native_bin_path):
print("running rust-native benchmark...\n{}".format(native_bin_path))
rust_cmd = shlex.split(native_bin_path)
stdoutlines = []
with subprocess.Popen(rust_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
bufsize=1,
universal_newlines=True) as p:
for line in p.stdout: # b'\n'-separated lines
print(line, end='')
stdoutlines.append(line) # pass bytes as is
p.wait()
exec_time_regex = "pairing check time: ([\.\w\d]+)"
exec_benchline = stdoutlines[-1]
exec_time_match = re.search(exec_time_regex, exec_benchline)
exec_us_time = durationpy.from_str(exec_time_match.group(1))
return {'exec_time': exec_us_time.total_seconds()}
def do_evmone_bench(evmone_bench_cmd):
evmone_cmd = shlex.split(evmone_bench_cmd)
print("running evmone benchmark...\n{}".format(evmone_bench_cmd))
stdoutlines = []
with subprocess.Popen(evmone_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
bufsize=1,
universal_newlines=True) as p:
for line in p.stdout: # b'\n'-separated lines
print(line, end='')
stdoutlines.append(line) # pass bytes as is
p.wait()
json_result = json.loads("".join(stdoutlines[2:]))
benchmarks = json_result['benchmarks']
benchmark_results = benchmarks[0]
gasused = int(benchmark_results['gas_used'])
total_time = str(
benchmark_results['real_time']) + benchmark_results['time_unit']
time = durationpy.from_str(total_time)
return {'gas_used': gasused, 'time': time.total_seconds()}
def do_evmone_bench(evmone_cmd):
print("running evmone benchmark...\n{}\n".format(evmone_cmd))
evmone_cmd = shlex.split(evmone_cmd)
stdoutlines = []
with subprocess.Popen(evmone_cmd,
cwd=EVMONE_BUILD_DIR,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
bufsize=1,
universal_newlines=True) as p:
for line in p.stdout: # b'\n'-separated lines
print(line, end='')
stdoutlines.append(line) # pass bytes as is
p.wait()
timeregex = "external_evm_code\s+(\d+) us"
gasregex = "gas_used=([\d\.\w]+)"
# maybe --benchmark_format=json is better so dont have to parse "36.775k"
benchline = stdoutlines[-1]
time_match = re.search(timeregex, benchline)
us_time = durationpy.from_str("{}us".format(time_match.group(1)))
gas_match = re.search(gasregex, benchline)
gasused = gas_match.group(1)
return {'gas_used': gasused, 'time': us_time.total_seconds()}
