def test_build_fuzzers_from_commit(self): """Tests if the fuzzers can build at a proper commit. This is done by using a known regression range for a specific test case. The old commit should show the error when its fuzzers run and the new one should not. """ test_data = os.path.join(TEST_DIR_PATH, 'testcases', 'yara_test_data') with tempfile.TemporaryDirectory() as tmp_dir: project_name = 'yara' old_commit = 'f79be4f2330f4b89ea2f42e1c44ca998c59a0c0f' new_commit = 'f50a39051ea8c7f10d6d8db9656658b49601caef' fuzzer = 'rules_fuzzer' yara_repo_manager = repo_manager.RepoManager( 'https://github.com/VirusTotal/yara.git', tmp_dir, repo_name='yara') build_data = build_specified_commit.BuildData() build_data.sanitizer = 'address' build_data.architecture = 'x86_64' build_data.engine = 'libfuzzer' build_data.project_name = 'yara' build_specified_commit.build_fuzzers_from_commit( old_commit, yara_repo_manager, build_data) old_error_code = helper.reproduce_impl(project_name, fuzzer, False, [], [], test_data) build_specified_commit.build_fuzzers_from_commit( new_commit, yara_repo_manager, build_data) new_error_code = helper.reproduce_impl(project_name, fuzzer, False, [], [], test_data) self.assertNotEqual(new_error_code, old_error_code)
def test_build_fuzzers_from_commit(self): """Tests if the fuzzers can build at a specified commit. This is done by using a known regression range for a specific test case. The old commit should show the error when its fuzzers run and the new one should not. """ with tempfile.TemporaryDirectory() as tmp_dir: test_case = test_repos.TEST_REPOS[1] self.assertTrue(helper.build_image_impl(test_case.project_name)) host_src_dir = build_specified_commit.copy_src_from_docker( test_case.project_name, tmp_dir) test_repo_manager = repo_manager.clone_repo_and_get_manager( test_case.git_url, host_src_dir, test_case.oss_repo_name) build_data = build_specified_commit.BuildData( sanitizer='address', architecture='x86_64', engine='libfuzzer', project_name=test_case.project_name) build_specified_commit.build_fuzzers_from_commit(test_case.old_commit, test_repo_manager, host_src_dir, build_data) old_error_code = helper.reproduce_impl(test_case.project_name, test_case.fuzz_target, False, [], [], test_case.test_case_path) build_specified_commit.build_fuzzers_from_commit(test_case.new_commit, test_repo_manager, host_src_dir, build_data) new_error_code = helper.reproduce_impl(test_case.project_name, test_case.fuzz_target, False, [], [], test_case.test_case_path) self.assertNotEqual(new_error_code, old_error_code)
def _bisect(old_commit, new_commit, test_case_path, fuzz_target, build_data): # pylint: disable=too-many-locals """Perform the bisect.""" with tempfile.TemporaryDirectory() as tmp_dir: repo_url, repo_path = build_specified_commit.detect_main_repo( build_data.project_name, commit=new_commit) if not repo_url or not repo_path: raise ValueError('Main git repo can not be determined.') # Copy /src from the built Docker container to ensure all dependencies # exist. This will be mounted when running them. host_src_dir = build_specified_commit.copy_src_from_docker( build_data.project_name, tmp_dir) bisect_repo_manager = repo_manager.BaseRepoManager( os.path.join(host_src_dir, os.path.basename(repo_path))) commit_list = bisect_repo_manager.get_commit_list(new_commit, old_commit) old_idx = len(commit_list) - 1 new_idx = 0 logging.info('Testing against new_commit (%s)', commit_list[new_idx]) if not build_specified_commit.build_fuzzers_from_commit( commit_list[new_idx], bisect_repo_manager, host_src_dir, build_data): raise RuntimeError('Failed to build new_commit') expected_error_code = helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], test_case_path) # Check if the error is persistent through the commit range if old_commit: logging.info('Testing against old_commit (%s)', commit_list[old_idx]) if not build_specified_commit.build_fuzzers_from_commit( commit_list[old_idx], bisect_repo_manager, host_src_dir, build_data, ): raise RuntimeError('Failed to build old_commit') if expected_error_code == helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], test_case_path): return Result(repo_url, commit_list[old_idx]) while old_idx - new_idx > 1: curr_idx = (old_idx + new_idx) // 2 logging.info('Testing against %s (idx=%d)', commit_list[curr_idx], curr_idx) build_specified_commit.build_fuzzers_from_commit(commit_list[curr_idx], bisect_repo_manager, host_src_dir, build_data) error_code = helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], test_case_path) if expected_error_code == error_code: new_idx = curr_idx else: old_idx = curr_idx return Result(repo_url, commit_list[new_idx])
def bisect(old_commit, new_commit, test_case_path, fuzz_target, build_data): """From a commit range, this function caluclates which introduced a specific error from a fuzz test_case_path. Args: old_commit: The oldest commit in the error regression range. new_commit: The newest commit in the error regression range. test_case_path: The file path of the test case that triggers the error fuzz_target: The name of the fuzzer to be tested. build_data: a class holding all of the input parameters for bisection. Returns: The commit SHA that introduced the error or None. Raises: ValueError: when a repo url can't be determine from the project. """ with tempfile.TemporaryDirectory() as tmp_dir: repo_url, repo_name = build_specified_commit.detect_main_repo( build_data.project_name, commit=old_commit) if not repo_url or not repo_name: raise ValueError('Main git repo can not be determined.') bisect_repo_manager = repo_manager.RepoManager(repo_url, tmp_dir, repo_name=repo_name) commit_list = bisect_repo_manager.get_commit_list( old_commit, new_commit) old_idx = len(commit_list) - 1 new_idx = 0 build_specified_commit.build_fuzzers_from_commit( commit_list[new_idx], bisect_repo_manager, build_data) expected_error_code = helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], test_case_path) # Check if the error is persistent through the commit range build_specified_commit.build_fuzzers_from_commit( commit_list[old_idx], bisect_repo_manager, build_data, ) if expected_error_code == helper.reproduce_impl( build_data.project_name, fuzz_target, False, [], [], test_case_path): return commit_list[old_idx] while old_idx - new_idx > 1: curr_idx = (old_idx + new_idx) // 2 build_specified_commit.build_fuzzers_from_commit( commit_list[curr_idx], bisect_repo_manager, build_data) error_code = helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], test_case_path) if expected_error_code == error_code: new_idx = curr_idx else: old_idx = curr_idx return commit_list[new_idx]
def build_fuzzers(args): """Builds all of the fuzzers for a specific OSS-Fuzz project. Returns: True on success False on failure. """ # TODO: Fix return value bubble to actually handle errors. with tempfile.TemporaryDirectory() as tmp_dir: inferred_url, repo_name = build_specified_commit.detect_main_repo( args.project_name, repo_name=args.repo_name) build_repo_manager = repo_manager.RepoManager(inferred_url, tmp_dir, repo_name=repo_name) build_data = build_specified_commit.BuildData() build_data.project_name = args.project_name build_data.sanitizer = 'address' build_data.engine = 'libfuzzer' build_data.architecture = 'x86_64' return build_specified_commit.build_fuzzers_from_commit( args.commit_sha, build_repo_manager, build_data) == 0
def bisect(old_commit, new_commit, test_case_path, fuzz_target, build_data): # pylint: disable=too-many-locals """From a commit range, this function caluclates which introduced a specific error from a fuzz test_case_path. Args: old_commit: The oldest commit in the error regression range. new_commit: The newest commit in the error regression range. test_case_path: The file path of the test case that triggers the error fuzz_target: The name of the fuzzer to be tested. build_data: a class holding all of the input parameters for bisection. Returns: The commit SHA that introduced the error or None. Raises: ValueError: when a repo url can't be determine from the project. """ with tempfile.TemporaryDirectory() as tmp_dir: repo_url, repo_path = build_specified_commit.detect_main_repo( build_data.project_name, commit=new_commit) if not repo_url or not repo_path: raise ValueError('Main git repo can not be determined.') host_src_dir = build_specified_commit.copy_src_from_docker( build_data.project_name, tmp_dir) bisect_repo_manager = repo_manager.RepoManager( repo_url, host_src_dir, repo_name=os.path.basename(repo_path)) commit_list = bisect_repo_manager.get_commit_list(new_commit, old_commit) old_idx = len(commit_list) - 1 new_idx = 0 logging.info('Testing against new_commit (%s)', commit_list[new_idx]) if not build_specified_commit.build_fuzzers_from_commit( commit_list[new_idx], bisect_repo_manager, host_src_dir, build_data): raise RuntimeError('Failed to build new_commit') expected_error_code = helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], test_case_path) # Check if the error is persistent through the commit range if old_commit: logging.info('Testing against old_commit (%s)', commit_list[old_idx]) if not build_specified_commit.build_fuzzers_from_commit( commit_list[old_idx], bisect_repo_manager, host_src_dir, build_data, ): raise RuntimeError('Failed to build old_commit') if expected_error_code == helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], test_case_path): return commit_list[old_idx] while old_idx - new_idx > 1: curr_idx = (old_idx + new_idx) // 2 logging.info('Testing against %s (idx=%d)', commit_list[curr_idx], curr_idx) build_specified_commit.build_fuzzers_from_commit(commit_list[curr_idx], bisect_repo_manager, host_src_dir, build_data) error_code = helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], test_case_path) if expected_error_code == error_code: new_idx = curr_idx else: old_idx = curr_idx return commit_list[new_idx]
def _bisect(bisect_type, old_commit, new_commit, test_case_path, fuzz_target, build_data): """Perform the bisect.""" # pylint: disable=too-many-branches base_builder_repo = build_specified_commit.load_base_builder_repo() with tempfile.TemporaryDirectory() as tmp_dir: repo_url, repo_path = build_specified_commit.detect_main_repo( build_data.project_name, commit=new_commit) if not repo_url or not repo_path: raise ValueError('Main git repo can not be determined.') if old_commit == new_commit: raise BisectError('old_commit is the same as new_commit', repo_url) # Copy /src from the built Docker container to ensure all dependencies # exist. This will be mounted when running them. host_src_dir = build_specified_commit.copy_src_from_docker( build_data.project_name, tmp_dir) bisect_repo_manager = repo_manager.BaseRepoManager( os.path.join(host_src_dir, os.path.basename(repo_path))) commit_list = bisect_repo_manager.get_commit_list( new_commit, old_commit) old_idx = len(commit_list) - 1 new_idx = 0 logging.info('Testing against new_commit (%s)', commit_list[new_idx]) if not build_specified_commit.build_fuzzers_from_commit( commit_list[new_idx], bisect_repo_manager, host_src_dir, build_data, base_builder_repo=base_builder_repo): raise BisectError('Failed to build new_commit', repo_url) if bisect_type == 'fixed': should_crash = False elif bisect_type == 'regressed': should_crash = True else: raise BisectError('Invalid bisect type ' + bisect_type, repo_url) expected_error = _check_for_crash(build_data.project_name, fuzz_target, test_case_path) logging.info('new_commit result = %s', expected_error) if not should_crash and expected_error: logging.warning('new_commit crashed but not shouldn\'t. ' 'Continuing to see if stack changes.') range_valid = False for _ in range(2): logging.info('Testing against old_commit (%s)', commit_list[old_idx]) if not build_specified_commit.build_fuzzers_from_commit( commit_list[old_idx], bisect_repo_manager, host_src_dir, build_data, base_builder_repo=base_builder_repo): raise BisectError('Failed to build old_commit', repo_url) if _check_for_crash(build_data.project_name, fuzz_target, test_case_path) == expected_error: logging.warning( 'old_commit %s had same result as new_commit %s', old_commit, new_commit) # Try again on an slightly older commit. old_commit = bisect_repo_manager.get_parent(old_commit, 64) if not old_commit: break commit_list = bisect_repo_manager.get_commit_list( new_commit, old_commit) old_idx = len(commit_list) - 1 continue range_valid = True break if not range_valid: raise BisectError('old_commit had same result as new_commit', repo_url) while old_idx - new_idx > 1: curr_idx = (old_idx + new_idx) // 2 logging.info('Testing against %s (idx=%d)', commit_list[curr_idx], curr_idx) if not build_specified_commit.build_fuzzers_from_commit( commit_list[curr_idx], bisect_repo_manager, host_src_dir, build_data, base_builder_repo=base_builder_repo): # Treat build failures as if we couldn't repo. # TODO(ochang): retry nearby commits? old_idx = curr_idx continue current_error = _check_for_crash(build_data.project_name, fuzz_target, test_case_path) logging.info('Current result = %s', current_error) if expected_error == current_error: new_idx = curr_idx else: old_idx = curr_idx return Result(repo_url, commit_list[new_idx])
def _bisect(bisect_type, old_commit, new_commit, test_case_path, fuzz_target, build_data): """Perform the bisect.""" # pylint: disable=too-many-branches base_builder_repo = _load_base_builder_repo() with tempfile.TemporaryDirectory() as tmp_dir: repo_url, repo_path = build_specified_commit.detect_main_repo( build_data.project_name, commit=new_commit) if not repo_url or not repo_path: raise ValueError('Main git repo can not be determined.') # Copy /src from the built Docker container to ensure all dependencies # exist. This will be mounted when running them. host_src_dir = build_specified_commit.copy_src_from_docker( build_data.project_name, tmp_dir) bisect_repo_manager = repo_manager.BaseRepoManager( os.path.join(host_src_dir, os.path.basename(repo_path))) commit_list = bisect_repo_manager.get_commit_list( new_commit, old_commit) old_idx = len(commit_list) - 1 new_idx = 0 logging.info('Testing against new_commit (%s)', commit_list[new_idx]) if not build_specified_commit.build_fuzzers_from_commit( commit_list[new_idx], bisect_repo_manager, host_src_dir, build_data, base_builder_repo=base_builder_repo): raise RuntimeError('Failed to build new_commit') if bisect_type == 'fixed': expected_error = False elif bisect_type == 'regressed': expected_error = True else: raise ValueError('Invalid bisect type ' + bisect_type) if _check_for_crash(build_data.project_name, fuzz_target, test_case_path) != expected_error: raise RuntimeError('new_commit didn\'t have expected result.') # Check if the error is persistent through the commit range if old_commit: logging.info('Testing against old_commit (%s)', commit_list[old_idx]) if not build_specified_commit.build_fuzzers_from_commit( commit_list[old_idx], bisect_repo_manager, host_src_dir, build_data, base_builder_repo=base_builder_repo): raise RuntimeError('Failed to build old_commit') if _check_for_crash(build_data.project_name, fuzz_target, test_case_path) == expected_error: raise RuntimeError('old_commit had same result as new_commit') while old_idx - new_idx > 1: curr_idx = (old_idx + new_idx) // 2 logging.info('Testing against %s (idx=%d)', commit_list[curr_idx], curr_idx) if not build_specified_commit.build_fuzzers_from_commit( commit_list[curr_idx], bisect_repo_manager, host_src_dir, build_data, base_builder_repo=base_builder_repo): # Treat build failures as if we couldn't repo. # TODO(ochang): retry nearby commits? old_idx = curr_idx continue is_error = _check_for_crash(build_data.project_name, fuzz_target, test_case_path) if expected_error == is_error: new_idx = curr_idx else: old_idx = curr_idx return Result(repo_url, commit_list[new_idx])