def test_infer_main_repo(self): """Tests that the main repo can be inferred based on an example commit.""" with tempfile.TemporaryDirectory() as tmp_dir: # Construct example repo's to check for commits. repo_manager.RepoManager('https://github.com/curl/curl.git', tmp_dir) repo_manager.RepoManager('https://github.com/weinrank/usrsctp', tmp_dir) repo_manager.RepoManager('https://github.com/ntop/nDPI.git', tmp_dir) repo_manager.RepoManager( 'https://github.com/libarchive/libarchive.git', tmp_dir) self.check_commit_with_repo( 'https://github.com/curl/curl.git', 'curl', 'bc5d22c3dede2f04870c37aec9a50474c4b888ad', tmp_dir) self.check_commit_with_repo( 'https://github.com/weinrank/usrsctp', 'usrsctp', '4886aaa49fb90e479226fcfc3241d74208908232', tmp_dir) self.check_commit_with_repo( 'https://github.com/ntop/nDPI.git', 'nDPI', 'c4d476cc583a2ef1e9814134efa4fbf484564ed7', tmp_dir) self.check_commit_with_repo( 'https://github.com/libarchive/libarchive.git', 'libarchive', '458e49358f17ec58d65ab1c45cf299baaf3c98d1', tmp_dir) self.check_commit_with_repo(None, None, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', tmp_dir)
def test_clone_correctly(self): """Tests the correct location of the git repo.""" test_repo_manager = repo_manager.RepoManager(self.curl_repo, 'tmp') git_path = os.path.join(test_repo_manager.base_dir, test_repo_manager.repo_name, '.git') self.assertTrue(os.path.isdir(git_path)) test_repo_manager.remove_repo() with self.assertRaises(repo_manager.RepoManagerError): test_repo_manager = repo_manager.RepoManager(' ', 'tmp')
def test_clone_invalid_repo(self): """Test that constructing RepoManager with an invalid repo will fail.""" with tempfile.TemporaryDirectory() as tmp_dir: with self.assertRaises(ValueError): repo_manager.RepoManager(' ', tmp_dir) with self.assertRaises(ValueError): repo_manager.RepoManager('not_a_valid_repo', tmp_dir) with self.assertRaises(ValueError): repo_manager.RepoManager( 'https://github.com/oss-fuzz-not-real.git', tmp_dir)
def test_clone_correctly(self): """Tests the correct location of the git repo.""" with tempfile.TemporaryDirectory() as tmp_dir: test_repo_manager = repo_manager.RepoManager( self.curl_repo, tmp_dir) git_path = os.path.join(test_repo_manager.base_dir, test_repo_manager.repo_name, '.git') self.assertTrue(os.path.isdir(git_path)) test_repo_manager.remove_repo() with self.assertRaises(repo_manager.RepoManagerError): test_repo_manager = repo_manager.RepoManager(' ', tmp_dir)
def test_infer_main_repo_from_name(self): """Tests that the main project repo can be inferred from a repo name.""" with tempfile.TemporaryDirectory() as tmp_dir: # Construct example repos to check for name. repo_manager.RepoManager('https://github.com/curl/curl.git', tmp_dir) repo_manager.RepoManager('https://github.com/ntop/nDPI.git', tmp_dir) repo_manager.RepoManager('https://github.com/libarchive/libarchive.git', tmp_dir) self.check_ref_with_repo('https://github.com/curl/curl.git', 'curl', tmp_dir) self.check_ref_with_repo('https://github.com/ntop/nDPI.git', 'nDPI', tmp_dir) self.check_ref_with_repo('https://github.com/libarchive/libarchive.git', 'libarchive', tmp_dir)
def test_diff_empty(self): """Tests that None is returned when there is no difference between repos.""" with tempfile.TemporaryDirectory() as tmp_dir: repo_man = repo_manager.RepoManager(OSS_FUZZ_REPO, tmp_dir) with mock.patch.object(utils, 'execute', return_value=('', None, 0)): diff = repo_man.get_git_diff() self.assertIsNone(diff)
def test_build_fuzzers_from_commit(self): """Tests if the fuzzers can build at a proper commit. This is done by using a known regression range for a specific test case. The old commit should show the error when its fuzzers run and the new one should not. """ test_data = os.path.join(TEST_DIR_PATH, 'testcases', 'yara_test_data') with tempfile.TemporaryDirectory() as tmp_dir: project_name = 'yara' old_commit = 'f79be4f2330f4b89ea2f42e1c44ca998c59a0c0f' new_commit = 'f50a39051ea8c7f10d6d8db9656658b49601caef' fuzzer = 'rules_fuzzer' yara_repo_manager = repo_manager.RepoManager( 'https://github.com/VirusTotal/yara.git', tmp_dir, repo_name='yara') build_data = build_specified_commit.BuildData() build_data.sanitizer = 'address' build_data.architecture = 'x86_64' build_data.engine = 'libfuzzer' build_data.project_name = 'yara' build_specified_commit.build_fuzzers_from_commit( old_commit, yara_repo_manager, build_data) old_error_code = helper.reproduce_impl(project_name, fuzzer, False, [], [], test_data) build_specified_commit.build_fuzzers_from_commit( new_commit, yara_repo_manager, build_data) new_error_code = helper.reproduce_impl(project_name, fuzzer, False, [], [], test_data) self.assertNotEqual(new_error_code, old_error_code)
def test_diff_empty(self): """Tests that None is returned when there is no difference between repos.""" with get_oss_fuzz_repo() as oss_fuzz_repo: repo_man = repo_manager.RepoManager(oss_fuzz_repo) with mock.patch.object(utils, 'execute', return_value=('', None, 0)): diff = repo_man.get_git_diff() self.assertIsNone(diff)
def test_build_fuzzers_from_commit(self): """Tests if the fuzzers can build at a proper commit. This is done by using a known regression range for a specific test case. The old commit should show the error when its fuzzers run and the new one should not. """ with tempfile.TemporaryDirectory() as tmp_dir: test_case = test_repos.TEST_REPOS[0] test_repo_manager = repo_manager.RepoManager( test_case.git_url, tmp_dir, repo_name=test_case.oss_repo_name) build_data = build_specified_commit.BuildData( sanitizer='address', architecture='x86_64', engine='libfuzzer', project_name=test_case.project_name) build_specified_commit.build_fuzzers_from_commit( test_case.old_commit, test_repo_manager, build_data) old_error_code = helper.reproduce_impl(test_case.project_name, test_case.fuzz_target, False, [], [], test_case.test_case_path) build_specified_commit.build_fuzzers_from_commit( test_case.new_commit, test_repo_manager, build_data) new_error_code = helper.reproduce_impl(test_case.project_name, test_case.fuzz_target, False, [], [], test_case.test_case_path) self.assertNotEqual(new_error_code, old_error_code)
def bisect(bisect_type, old_commit, new_commit, testcase_path, fuzz_target, build_data): """From a commit range, this function caluclates which introduced a specific error from a fuzz testcase_path. Args: bisect_type: The type of the bisect ('regressed' or 'fixed'). old_commit: The oldest commit in the error regression range. new_commit: The newest commit in the error regression range. testcase_path: The file path of the test case that triggers the error fuzz_target: The name of the fuzzer to be tested. build_data: a class holding all of the input parameters for bisection. Returns: The commit SHA that introduced the error or None. Raises: ValueError: when a repo url can't be determine from the project. """ try: return _bisect(bisect_type, old_commit, new_commit, testcase_path, fuzz_target, build_data) finally: # Clean up projects/ as _bisect may have modified it. oss_fuzz_repo_manager = repo_manager.RepoManager(helper.OSS_FUZZ_DIR) oss_fuzz_repo_manager.git(['reset', 'projects']) oss_fuzz_repo_manager.git(['checkout', 'projects']) oss_fuzz_repo_manager.git(['clean', '-fxd', 'projects'])
def build_fuzzer_from_commit(project_name, commit, local_store_path, engine='libfuzzer', sanitizer='address', architecture='x86_64', old_repo_manager=None): """Builds a OSS-Fuzz fuzzer at a specific commit SHA. Args: project_name: The OSS-Fuzz project name commit: The commit SHA to build the fuzzers at local_store_path: The full file path of a place where a temp git repo is stored engine: The fuzzing engine to be used sanitizer: The fuzzing sanitizer to be used architecture: The system architiecture to be used for fuzzing Returns: 0 on successful build 1 on failure """ if not old_repo_manager: inferred_url = infer_main_repo(project_name, local_store_path, commit) old_repo_manager = repo_manager.RepoManager(inferred_url, local_store_path) old_repo_manager.checkout_commit(commit) return helper.build_fuzzers_impl(project_name=project_name, clean=True, engine=engine, sanitizer=sanitizer, architecture=architecture, env_to_add=None, source_path=old_repo_manager.repo_dir)
def test_checkout_valid_commit(self): """Tests that the git checkout command works.""" with tempfile.TemporaryDirectory() as tmp_dir: test_repo_manager = repo_manager.RepoManager(OSS_FUZZ_REPO, tmp_dir) commit_to_test = '04ea24ee15bbe46a19e5da6c5f022a2ffdfbdb3b' test_repo_manager.checkout_commit(commit_to_test) self.assertEqual(commit_to_test, test_repo_manager.get_current_commit())
def test_checkout_valid_pull_request(self): """Tests that the git checkout pull request works.""" with tempfile.TemporaryDirectory() as tmp_dir: test_repo_manager = repo_manager.RepoManager(OSS_FUZZ_REPO, tmp_dir) test_repo_manager.checkout_pr('refs/pull/3415/merge') self.assertEqual(test_repo_manager.get_current_commit(), '314c9249a54a08e764a5bbcb7333294ae7c1f9ed')
def bisect(commit_old, commit_new, testcase, fuzz_target, build_data): """From a commit range, this function caluclates which introduced a specific error from a fuzz testcase. Args: commit_old: The oldest commit in the error regression range commit_new: The newest commit in the error regression range testcase: The file path of the test case that triggers the error fuzz_target: The name of the fuzzer to be tested build_data: a class holding all of the input parameters for bisection Returns: The commit SHA that introduced the error or None Raises: ValueError: when a repo url can't be determine from the project """ with tempfile.TemporaryDirectory() as tmp_dir: repo_url, repo_name = build_specified_commit.detect_main_repo_from_docker( build_data.project_name, commit_old) if not repo_url or not repo_name: raise ValueError('Main git repo can not be determined.') bisect_repo_manager = repo_manager.RepoManager( repo_url, tmp_dir, repo_name=repo_name) commit_list = bisect_repo_manager.get_commit_list(commit_old, commit_new) old_idx = len(commit_list) - 1 new_idx = 0 build_specified_commit.build_fuzzer_from_commit( build_data.project_name, commit_list[new_idx], bisect_repo_manager.repo_dir, build_data.engine, build_data.sanitizer, build_data.architecture, bisect_repo_manager) expected_error_code = helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], testcase) # Check if the error is persistent through the commit range build_specified_commit.build_fuzzer_from_commit( build_data.project_name, commit_list[old_idx], bisect_repo_manager.repo_dir, build_data.engine, build_data.sanitizer, build_data.architecture, bisect_repo_manager) oldest_error_code = helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], testcase) if expected_error_code == oldest_error_code: return commit_list[old_idx] while old_idx - new_idx > 1: curr_idx = (old_idx + new_idx) // 2 build_specified_commit.build_fuzzer_from_commit( build_data.project_name, commit_list[curr_idx], bisect_repo_manager.repo_dir, build_data.engine, build_data.sanitizer, build_data.architecture, bisect_repo_manager) error_code = helper.reproduce_impl(build_data.project_name, fuzz_target, False, [], [], testcase) if expected_error_code == error_code: new_idx = curr_idx else: old_idx = curr_idx return commit_list[new_idx]
def test_pull_request_exists(self): """Tests that a diff is returned when a valid PR is checked out.""" with get_oss_fuzz_repo() as oss_fuzz_repo: repo_man = repo_manager.RepoManager(oss_fuzz_repo) repo_man.checkout_pr('refs/pull/3415/merge') diff = repo_man.get_git_diff() self.assertCountEqual(diff, ['README.md'])
def test_checkout_valid_commit(self): """Tests that the git checkout command works.""" with get_oss_fuzz_repo() as oss_fuzz_repo: repo_man = repo_manager.RepoManager(oss_fuzz_repo) commit_to_test = '04ea24ee15bbe46a19e5da6c5f022a2ffdfbdb3b' repo_man.checkout_commit(commit_to_test) self.assertEqual(commit_to_test, repo_man.get_current_commit())
def test_clone_valid_repo(self): """Tests the correct location of the git repo.""" with tempfile.TemporaryDirectory() as tmp_dir: test_repo_manager = repo_manager.RepoManager(OSS_FUZZ_REPO, tmp_dir) git_path = os.path.join(test_repo_manager.base_dir, test_repo_manager.repo_name, '.git') self.assertTrue(os.path.isdir(git_path)) test_repo_manager.remove_repo()
def test_pull_request_exists(self): """Tests that a diff is returned when a valid PR is checked out.""" with tempfile.TemporaryDirectory() as tmp_dir: repo_man = repo_manager.RepoManager(OSS_FUZZ_REPO, tmp_dir) repo_man.checkout_pr('refs/pull/3415/merge') diff = repo_man.get_git_diff() print(diff) self.assertCountEqual(diff, ['README.md'])
def test_checkout_valid_pull_request(self): """Tests that the git checkout pull request works.""" with tempfile.TemporaryDirectory() as tmp_dir: test_repo_manager = repo_manager.RepoManager( OSS_FUZZ_REPO, tmp_dir) test_repo_manager.checkout_pr('refs/pull/3310/merge') self.assertEqual(test_repo_manager.get_current_commit(), 'ff00c1685ccf32f729cf6c834e641223ce6262e4')
def test_checkout_valid_pull_request(self): """Tests that the git checkout pull request works.""" with tempfile.TemporaryDirectory() as tmp_dir: test_repo_manager = repo_manager.RepoManager( OSS_FUZZ_REPO, tmp_dir) test_repo_manager.checkout_pr('refs/pull/1757/merge') self.assertEqual(test_repo_manager.get_current_commit(), '2a2b11cc3d370db8f7bdf73046f3290a39615347')
def test_infer_main_repo_from_name(self): """Tests that the main project repo can be inferred from a repo name.""" with tempfile.TemporaryDirectory() as tmp_dir: for example_repo in test_repos.TEST_REPOS: repo_manager.RepoManager(example_repo.git_url, tmp_dir) self.check_with_repo(example_repo.git_url, example_repo.git_repo_name, tmp_dir)
def test_diff_exists(self): """Tests that a real diff is returned when a valid repo manager exists.""" with get_oss_fuzz_repo() as oss_fuzz_repo: repo_man = repo_manager.RepoManager(oss_fuzz_repo) with mock.patch.object(utils, 'execute', return_value=('test.py\ndiff.py', None, 0)): diff = repo_man.get_git_diff() self.assertCountEqual(diff, ['test.py', 'diff.py'])
def test_diff_exists(self): """Tests that a real diff is returned when a valid repo manager exists.""" with tempfile.TemporaryDirectory() as tmp_dir: repo_man = repo_manager.RepoManager(OSS_FUZZ_REPO, tmp_dir) with mock.patch.object(utils, 'execute', return_value=('test.py\ndiff.py', None, 0)): diff = repo_man.get_git_diff() self.assertCountEqual(diff, ['test.py', 'diff.py'])
def test_error_on_command(self): """Tests that None is returned when the command errors out.""" with tempfile.TemporaryDirectory() as tmp_dir: repo_man = repo_manager.RepoManager(OSS_FUZZ_REPO, tmp_dir) with mock.patch.object(utils, 'execute', return_value=('', 'Test error.', 1)): diff = repo_man.get_git_diff() self.assertIsNone(diff)
def test_error_on_command(self): """Tests that None is returned when the command errors out.""" with get_oss_fuzz_repo() as oss_fuzz_repo: repo_man = repo_manager.RepoManager(oss_fuzz_repo) with mock.patch.object(utils, 'execute', return_value=('', 'Test error.', 1)): diff = repo_man.get_git_diff() self.assertIsNone(diff)
def test_checkout_invalid_commit(self): """Tests that the git checkout invalid commit fails.""" with get_oss_fuzz_repo() as oss_fuzz_repo: repo_man = repo_manager.RepoManager(oss_fuzz_repo) with self.assertRaises(ValueError): repo_man.checkout_commit(' ') with self.assertRaises(ValueError): repo_man.checkout_commit('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') with self.assertRaises(ValueError): repo_man.checkout_commit('not-a-valid-commit')
def test_checkout_invalid_pull_request(self): """Tests that the git checkout invalid pull request fails.""" with get_oss_fuzz_repo() as oss_fuzz_repo: repo_man = repo_manager.RepoManager(oss_fuzz_repo) with self.assertRaises(RuntimeError): repo_man.checkout_pr(' ') with self.assertRaises(RuntimeError): repo_man.checkout_pr('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') with self.assertRaises(RuntimeError): repo_man.checkout_pr('not/a/valid/pr')
def test_checkout_invalid_commit(self): """Tests that the git checkout invalid commit fails.""" with tempfile.TemporaryDirectory() as tmp_dir: test_repo_manager = repo_manager.RepoManager(OSS_FUZZ_REPO, tmp_dir) with self.assertRaises(ValueError): test_repo_manager.checkout_commit(' ') with self.assertRaises(ValueError): test_repo_manager.checkout_commit( 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') with self.assertRaises(ValueError): test_repo_manager.checkout_commit('not-a-valid-commit')
def test_checkout_invalid_pull_request(self): """Tests that the git checkout invalid pull request fails.""" with tempfile.TemporaryDirectory() as tmp_dir: test_repo_manager = repo_manager.RepoManager(OSS_FUZZ_REPO, tmp_dir) with self.assertRaises(RuntimeError): test_repo_manager.checkout_pr(' ') with self.assertRaises(RuntimeError): test_repo_manager.checkout_pr( 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') with self.assertRaises(RuntimeError): test_repo_manager.checkout_pr('not/a/valid/pr')
def test_fix_git_repo_for_diff(self, mocked_execute): """Tests that fix_git_repo_for_diff works as intended.""" repo_dir = '/dir' repo_manager_obj = repo_manager.RepoManager(repo_dir) continuous_integration.fix_git_repo_for_diff(repo_manager_obj) expected_command = [ 'git', 'symbolic-ref', 'refs/remotes/origin/HEAD', 'refs/remotes/origin/master' ] mocked_execute.assert_called_with(expected_command, location=repo_dir)