def test_bench(): """Runs benchmarks before and after and compares the results.""" os.chdir(get_repo_root_path()) # Get numbers for current HEAD. return_code, stdout, stderr = _run_cargo_bench(PR_BENCH_RESULTS_FILE) # Even if it is the first time this test is run, the benchmark tests should pass. # For this purpose, we need to explicitly check the return code. assert return_code == 0, "stdout: {}\n stderr: {}".format(stdout, stderr) # Get numbers from upstream tip, without the changes from the current PR. _git_checkout_upstream_branch() return_code, stdout, stderr = _run_cargo_bench(UPSTREAM_BENCH_RESULTS_FILE) # Before checking any results, let's just go back to the PR branch. # This way we make sure that the cleanup always happens even if the test fails. _git_checkout_pr_branch() if return_code == 0: # In case this benchmark also ran successfully, we can call critcmp and compare the results. _run_critcmp() else: # The benchmark did not run successfully, but it might be that it is because a benchmark does not exist. # In this case, we do not want to fail the test. if "error: no bench target named `main`" in stderr: # This is a bit of a &*%^ way of checking if the benchmark does not exist. # Hopefully it will be possible to check it in another way...soon print( "There are no benchmarks in master. No comparison can happen.") else: assert return_code == 0, "stdout: {}\n stderr: {}".format( stdout, stderr)
# up wrongfully using the config file in the rust-vmm-ci submodule. # os.walkdir() offers a depth-first search and couldn't be used here. dirs = [os.getcwd()] while len(dirs): nextDirs = [] for dir in dirs: for file in os.listdir(dir): file_path = os.path.join(dir, file) if os.path.isdir(file_path): nextDirs.append(file_path) elif file == target_file: return file_path dirs = nextDirs REPO_ROOT_PATH = get_repo_root_path() COVERAGE_CONFIG_PATH = get_coverage_config_path() def _read_test_config(): """ Reads the config of the coverage for the repository being tested. Returns a JSON object with the configuration. """ coverage_config = {} with open(COVERAGE_CONFIG_PATH) as config_file: coverage_config = json.load(config_file) assert "coverage_score" in coverage_config assert "exclude_path" in coverage_config