def test_build_fuzzers_from_commit(self): """Tests if the fuzzers can build at a proper commit. This is done by using a known regression range for a specific test case. The old commit should show the error when its fuzzers run and the new one should not. """ test_data = os.path.join(TEST_DIR_PATH, 'testcases', 'yara_test_data') with tempfile.TemporaryDirectory() as tmp_dir: project_name = 'yara' old_commit = 'f79be4f2330f4b89ea2f42e1c44ca998c59a0c0f' new_commit = 'f50a39051ea8c7f10d6d8db9656658b49601caef' fuzzer = 'rules_fuzzer' yara_repo_manager = repo_manager.RepoManager( 'https://github.com/VirusTotal/yara.git', tmp_dir, repo_name='yara') build_data = build_specified_commit.BuildData() build_data.sanitizer = 'address' build_data.architecture = 'x86_64' build_data.engine = 'libfuzzer' build_data.project_name = 'yara' build_specified_commit.build_fuzzers_from_commit( old_commit, yara_repo_manager, build_data) old_error_code = helper.reproduce_impl(project_name, fuzzer, False, [], [], test_data) build_specified_commit.build_fuzzers_from_commit( new_commit, yara_repo_manager, build_data) new_error_code = helper.reproduce_impl(project_name, fuzzer, False, [], [], test_data) self.assertNotEqual(new_error_code, old_error_code)
def test_build_fuzzers_from_commit(self): """Tests if the fuzzers can build at a specified commit. This is done by using a known regression range for a specific test case. The old commit should show the error when its fuzzers run and the new one should not. """ with tempfile.TemporaryDirectory() as tmp_dir: test_case = test_repos.TEST_REPOS[1] self.assertTrue(helper.build_image_impl(test_case.project_name)) host_src_dir = build_specified_commit.copy_src_from_docker( test_case.project_name, tmp_dir) test_repo_manager = repo_manager.clone_repo_and_get_manager( test_case.git_url, host_src_dir, test_case.oss_repo_name) build_data = build_specified_commit.BuildData( sanitizer='address', architecture='x86_64', engine='libfuzzer', project_name=test_case.project_name) build_specified_commit.build_fuzzers_from_commit(test_case.old_commit, test_repo_manager, host_src_dir, build_data) old_error_code = helper.reproduce_impl(test_case.project_name, test_case.fuzz_target, False, [], [], test_case.test_case_path) build_specified_commit.build_fuzzers_from_commit(test_case.new_commit, test_repo_manager, host_src_dir, build_data) new_error_code = helper.reproduce_impl(test_case.project_name, test_case.fuzz_target, False, [], [], test_case.test_case_path) self.assertNotEqual(new_error_code, old_error_code)
def bisect(commit_old, commit_new, testcase, fuzz_target, build_data): """From a commit range, this function caluclates which introduced a specific error from a fuzz testcase. Args: commit_old: The oldest commit in the error regression range commit_new: The newest commit in the error regression range testcase: The file path of the test case that triggers the error fuzz_target: The name of the fuzzer to be tested build_data: a class holding all of the input parameters for bisection Returns: The commit SHA that introduced the error or None Raises: ValueError: when a repo url can't be determine from the project """ with tempfile.TemporaryDirectory() as tmp_dir: repo_url, repo_name = build_specified_commit.detect_main_repo_from_docker( build_data.project_name, commit_old) if not repo_url or not repo_name: raise ValueError('Main git repo can not be determined.') bisect_repo_manager = repo_manager.RepoManager( repo_url, tmp_dir, repo_name=repo_name) commit_list = bisect_repo_manager.get_commit_list(commit_old, commit_new) old_idx = len(commit_list) - 1 new_idx = 0 build_specified_commit.build_fuzzer_from_commit( build_data.project_name, commit_list[new_idx], bisect_repo_manager.repo_dir, build_data.engine, build_data.sanitizer, build_data.architecture, bisect_repo_manager) expected_error_code = helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], testcase) # Check if the error is persistent through the commit range build_specified_commit.build_fuzzer_from_commit( build_data.project_name, commit_list[old_idx], bisect_repo_manager.repo_dir, build_data.engine, build_data.sanitizer, build_data.architecture, bisect_repo_manager) oldest_error_code = helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], testcase) if expected_error_code == oldest_error_code: return commit_list[old_idx] while old_idx - new_idx > 1: curr_idx = (old_idx + new_idx) // 2 build_specified_commit.build_fuzzer_from_commit( build_data.project_name, commit_list[curr_idx], bisect_repo_manager.repo_dir, build_data.engine, build_data.sanitizer, build_data.architecture, bisect_repo_manager) error_code = helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], testcase) if expected_error_code == error_code: new_idx = curr_idx else: old_idx = curr_idx return commit_list[new_idx]
def _bisect(old_commit, new_commit, test_case_path, fuzz_target, build_data): # pylint: disable=too-many-locals """Perform the bisect.""" with tempfile.TemporaryDirectory() as tmp_dir: repo_url, repo_path = build_specified_commit.detect_main_repo( build_data.project_name, commit=new_commit) if not repo_url or not repo_path: raise ValueError('Main git repo can not be determined.') # Copy /src from the built Docker container to ensure all dependencies # exist. This will be mounted when running them. host_src_dir = build_specified_commit.copy_src_from_docker( build_data.project_name, tmp_dir) bisect_repo_manager = repo_manager.BaseRepoManager( os.path.join(host_src_dir, os.path.basename(repo_path))) commit_list = bisect_repo_manager.get_commit_list(new_commit, old_commit) old_idx = len(commit_list) - 1 new_idx = 0 logging.info('Testing against new_commit (%s)', commit_list[new_idx]) if not build_specified_commit.build_fuzzers_from_commit( commit_list[new_idx], bisect_repo_manager, host_src_dir, build_data): raise RuntimeError('Failed to build new_commit') expected_error_code = helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], test_case_path) # Check if the error is persistent through the commit range if old_commit: logging.info('Testing against old_commit (%s)', commit_list[old_idx]) if not build_specified_commit.build_fuzzers_from_commit( commit_list[old_idx], bisect_repo_manager, host_src_dir, build_data, ): raise RuntimeError('Failed to build old_commit') if expected_error_code == helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], test_case_path): return Result(repo_url, commit_list[old_idx]) while old_idx - new_idx > 1: curr_idx = (old_idx + new_idx) // 2 logging.info('Testing against %s (idx=%d)', commit_list[curr_idx], curr_idx) build_specified_commit.build_fuzzers_from_commit(commit_list[curr_idx], bisect_repo_manager, host_src_dir, build_data) error_code = helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], test_case_path) if expected_error_code == error_code: new_idx = curr_idx else: old_idx = curr_idx return Result(repo_url, commit_list[new_idx])
def _check_for_crash(project_name, fuzz_target, test_case_path): """Check for crash.""" def docker_run(args): command = ['docker', 'run', '--rm', '--privileged'] if sys.stdin.isatty(): command.append('-i') return utils.execute(command + args) logging.info('Checking for crash') out, err, return_code = helper.reproduce_impl(project_name, fuzz_target, False, [], [], test_case_path, runner=docker_run, err_result=(None, None, None)) if return_code is None: return None logging.info('stdout =\n%s', out) logging.info('stderr =\n%s', err) # pylint: disable=unsupported-membership-test has_start_marker = any(marker in out or marker in err for marker in START_MARKERS) has_end_marker = any(marker in out or marker in err for marker in END_MARKERS) if not has_start_marker or not has_end_marker: return None return _get_dedup_token(out + err)
def bisect(commit_old, commit_new, testcase, fuzz_target, build_data): """From a commit range, this function caluclates which introduced a specific error from a fuzz testcase. Args: commit_old: The oldest commit in the error regression range commit_new: The newest commit in the error regression range testcase: The file path of the test case that triggers the error fuzz_target: The name of the fuzzer to be tested build_data: a class holding all of the input parameters for bisection Returns: The commit SHA that introduced the error or None """ local_store_path = tempfile.mkdtemp() repo_url = build_specified_commit.infer_main_repo(build_data.project_name, local_store_path, commit_old) bisect_repo_manager = repo_manager.RepoManager(repo_url, local_store_path) commit_list = bisect_repo_manager.get_commit_list(commit_old, commit_new) build_specified_commit.build_fuzzer_from_commit( build_data.project_name, commit_list[0], bisect_repo_manager.repo_dir, build_data.engine, build_data.sanitizer, build_data.architecture, bisect_repo_manager) error_code = helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], testcase) old_idx = len(commit_list) - 1 new_idx = 0 if len(commit_list) == 1: if not error_code: return None return commit_list[0] while old_idx - new_idx != 1: curr_idx = (old_idx + new_idx) // 2 build_specified_commit.build_fuzzer_from_commit( build_data.project_name, commit_list[curr_idx], bisect_repo_manager.repo_dir, build_data.engine, build_data.sanitizer, build_data.architecture, bisect_repo_manager) error_exists = ( helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], testcase) == error_code) if error_exists == error_code: new_idx = curr_idx else: old_idx = curr_idx return commit_list[new_idx]
def test_build_fuzzers_from_commit(self): """Tests if the fuzzers can build at a proper commit. This is done by using a known regression range for a specific test case. The old commit should show the error when its fuzzers run and the new one should not. """ project_name = 'yara' old_commit = 'f79be4f2330f4b89ea2f42e1c44ca998c59a0c0f' new_commit = 'f50a39051ea8c7f10d6d8db9656658b49601caef' fuzzer = 'rules_fuzzer' test_data = 'infra/yara_test_data' build_specified_commit.build_fuzzer_from_commit( project_name, old_commit, 'tmp', sanitizer='address') old_error_code = helper.reproduce_impl(project_name, fuzzer, False, [], [], test_data) build_specified_commit.build_fuzzer_from_commit( project_name, new_commit, 'tmp', sanitizer='address') new_error_code = helper.reproduce_impl(project_name, fuzzer, False, [], [], test_data) self.assertNotEqual(new_error_code, old_error_code)
def bisect(old_commit, new_commit, test_case_path, fuzz_target, build_data): # pylint: disable=too-many-locals """From a commit range, this function caluclates which introduced a specific error from a fuzz test_case_path. Args: old_commit: The oldest commit in the error regression range. new_commit: The newest commit in the error regression range. test_case_path: The file path of the test case that triggers the error fuzz_target: The name of the fuzzer to be tested. build_data: a class holding all of the input parameters for bisection. Returns: The commit SHA that introduced the error or None. Raises: ValueError: when a repo url can't be determine from the project. """ with tempfile.TemporaryDirectory() as tmp_dir: repo_url, repo_path = build_specified_commit.detect_main_repo( build_data.project_name, commit=new_commit) if not repo_url or not repo_path: raise ValueError('Main git repo can not be determined.') host_src_dir = build_specified_commit.copy_src_from_docker( build_data.project_name, tmp_dir) bisect_repo_manager = repo_manager.RepoManager( repo_url, host_src_dir, repo_name=os.path.basename(repo_path)) commit_list = bisect_repo_manager.get_commit_list(new_commit, old_commit) old_idx = len(commit_list) - 1 new_idx = 0 logging.info('Testing against new_commit (%s)', commit_list[new_idx]) if not build_specified_commit.build_fuzzers_from_commit( commit_list[new_idx], bisect_repo_manager, host_src_dir, build_data): raise RuntimeError('Failed to build new_commit') expected_error_code = helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], test_case_path) # Check if the error is persistent through the commit range if old_commit: logging.info('Testing against old_commit (%s)', commit_list[old_idx]) if not build_specified_commit.build_fuzzers_from_commit( commit_list[old_idx], bisect_repo_manager, host_src_dir, build_data, ): raise RuntimeError('Failed to build old_commit') if expected_error_code == helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], test_case_path): return commit_list[old_idx] while old_idx - new_idx > 1: curr_idx = (old_idx + new_idx) // 2 logging.info('Testing against %s (idx=%d)', commit_list[curr_idx], curr_idx) build_specified_commit.build_fuzzers_from_commit(commit_list[curr_idx], bisect_repo_manager, host_src_dir, build_data) error_code = helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], test_case_path) if expected_error_code == error_code: new_idx = curr_idx else: old_idx = curr_idx return commit_list[new_idx]