def test_bench():
"""Runs benchmarks before and after and compares the results."""
os.chdir(get_repo_root_path())
# Get numbers for current HEAD.
return_code, stdout, stderr = _run_cargo_bench(PR_BENCH_RESULTS_FILE)
# Even if it is the first time this test is run, the benchmark tests should pass.
# For this purpose, we need to explicitly check the return code.
assert return_code == 0, "stdout: {}\n stderr: {}".format(stdout, stderr)
# Get numbers from upstream tip, without the changes from the current PR.
_git_checkout_upstream_branch()
return_code, stdout, stderr = _run_cargo_bench(UPSTREAM_BENCH_RESULTS_FILE)
# Before checking any results, let's just go back to the PR branch.
# This way we make sure that the cleanup always happens even if the test fails.
_git_checkout_pr_branch()
if return_code == 0:
# In case this benchmark also ran successfully, we can call critcmp and compare the results.
_run_critcmp()
else:
# The benchmark did not run successfully, but it might be that it is because a benchmark does not exist.
# In this case, we do not want to fail the test.
if "error: no bench target named `main`" in stderr:
# This is a bit of a &*%^ way of checking if the benchmark does not exist.
# Hopefully it will be possible to check it in another way...soon
print(
"There are no benchmarks in master. No comparison can happen.")
else:
assert return_code == 0, "stdout: {}\n stderr: {}".format(
stdout, stderr)
# up wrongfully using the config file in the rust-vmm-ci submodule.
# os.walkdir() offers a depth-first search and couldn't be used here.
dirs = [os.getcwd()]
while len(dirs):
nextDirs = []
for dir in dirs:
for file in os.listdir(dir):
file_path = os.path.join(dir, file)
if os.path.isdir(file_path):
nextDirs.append(file_path)
elif file == target_file:
return file_path
dirs = nextDirs
REPO_ROOT_PATH = get_repo_root_path()
COVERAGE_CONFIG_PATH = get_coverage_config_path()
def _read_test_config():
"""
Reads the config of the coverage for the repository being tested.
Returns a JSON object with the configuration.
"""
coverage_config = {}
with open(COVERAGE_CONFIG_PATH) as config_file:
coverage_config = json.load(config_file)
assert "coverage_score" in coverage_config
assert "exclude_path" in coverage_config