def test_build_fuzzers_from_commit(self):
"""Tests if the fuzzers can build at a proper commit.
This is done by using a known regression range for a specific test case.
The old commit should show the error when its fuzzers run and the new one
should not.
"""
test_data = os.path.join(TEST_DIR_PATH, 'testcases', 'yara_test_data')
with tempfile.TemporaryDirectory() as tmp_dir:
project_name = 'yara'
old_commit = 'f79be4f2330f4b89ea2f42e1c44ca998c59a0c0f'
new_commit = 'f50a39051ea8c7f10d6d8db9656658b49601caef'
fuzzer = 'rules_fuzzer'
yara_repo_manager = repo_manager.RepoManager(
'https://github.com/VirusTotal/yara.git',
tmp_dir,
repo_name='yara')
build_data = build_specified_commit.BuildData()
build_data.sanitizer = 'address'
build_data.architecture = 'x86_64'
build_data.engine = 'libfuzzer'
build_data.project_name = 'yara'
build_specified_commit.build_fuzzers_from_commit(
old_commit, yara_repo_manager, build_data)
old_error_code = helper.reproduce_impl(project_name, fuzzer, False,
[], [], test_data)
build_specified_commit.build_fuzzers_from_commit(
new_commit, yara_repo_manager, build_data)
new_error_code = helper.reproduce_impl(project_name, fuzzer, False,
[], [], test_data)
self.assertNotEqual(new_error_code, old_error_code)
def test_build_fuzzers_from_commit(self):
"""Tests if the fuzzers can build at a specified commit.
This is done by using a known regression range for a specific test case.
The old commit should show the error when its fuzzers run and the new one
should not.
"""
with tempfile.TemporaryDirectory() as tmp_dir:
test_case = test_repos.TEST_REPOS[1]
self.assertTrue(helper.build_image_impl(test_case.project_name))
host_src_dir = build_specified_commit.copy_src_from_docker(
test_case.project_name, tmp_dir)
test_repo_manager = repo_manager.clone_repo_and_get_manager(
test_case.git_url, host_src_dir, test_case.oss_repo_name)
build_data = build_specified_commit.BuildData(
sanitizer='address',
architecture='x86_64',
engine='libfuzzer',
project_name=test_case.project_name)
build_specified_commit.build_fuzzers_from_commit(test_case.old_commit,
test_repo_manager,
host_src_dir, build_data)
old_error_code = helper.reproduce_impl(test_case.project_name,
test_case.fuzz_target, False, [],
[], test_case.test_case_path)
build_specified_commit.build_fuzzers_from_commit(test_case.new_commit,
test_repo_manager,
host_src_dir, build_data)
new_error_code = helper.reproduce_impl(test_case.project_name,
test_case.fuzz_target, False, [],
[], test_case.test_case_path)
self.assertNotEqual(new_error_code, old_error_code)
def bisect(commit_old, commit_new, testcase, fuzz_target, build_data):
"""From a commit range, this function caluclates which introduced a
specific error from a fuzz testcase.
Args:
commit_old: The oldest commit in the error regression range
commit_new: The newest commit in the error regression range
testcase: The file path of the test case that triggers the error
fuzz_target: The name of the fuzzer to be tested
build_data: a class holding all of the input parameters for bisection
Returns:
The commit SHA that introduced the error or None
Raises:
ValueError: when a repo url can't be determine from the project
"""
with tempfile.TemporaryDirectory() as tmp_dir:
repo_url, repo_name = build_specified_commit.detect_main_repo_from_docker(
build_data.project_name, commit_old)
if not repo_url or not repo_name:
raise ValueError('Main git repo can not be determined.')
bisect_repo_manager = repo_manager.RepoManager(
repo_url, tmp_dir, repo_name=repo_name)
commit_list = bisect_repo_manager.get_commit_list(commit_old, commit_new)
old_idx = len(commit_list) - 1
new_idx = 0
build_specified_commit.build_fuzzer_from_commit(
build_data.project_name, commit_list[new_idx],
bisect_repo_manager.repo_dir, build_data.engine, build_data.sanitizer,
build_data.architecture, bisect_repo_manager)
expected_error_code = helper.reproduce_impl(build_data.project_name,
fuzz_target, False, [], [],
testcase)
# Check if the error is persistent through the commit range
build_specified_commit.build_fuzzer_from_commit(
build_data.project_name, commit_list[old_idx],
bisect_repo_manager.repo_dir, build_data.engine, build_data.sanitizer,
build_data.architecture, bisect_repo_manager)
oldest_error_code = helper.reproduce_impl(build_data.project_name,
fuzz_target, False, [], [],
testcase)
if expected_error_code == oldest_error_code:
return commit_list[old_idx]
while old_idx - new_idx > 1:
curr_idx = (old_idx + new_idx) // 2
build_specified_commit.build_fuzzer_from_commit(
build_data.project_name, commit_list[curr_idx],
bisect_repo_manager.repo_dir, build_data.engine, build_data.sanitizer,
build_data.architecture, bisect_repo_manager)
error_code = helper.reproduce_impl(build_data.project_name, fuzz_target,
False, [], [], testcase)
if expected_error_code == error_code:
new_idx = curr_idx
else:
old_idx = curr_idx
return commit_list[new_idx]
def _bisect(old_commit, new_commit, test_case_path, fuzz_target, build_data): # pylint: disable=too-many-locals
"""Perform the bisect."""
with tempfile.TemporaryDirectory() as tmp_dir:
repo_url, repo_path = build_specified_commit.detect_main_repo(
build_data.project_name, commit=new_commit)
if not repo_url or not repo_path:
raise ValueError('Main git repo can not be determined.')
# Copy /src from the built Docker container to ensure all dependencies
# exist. This will be mounted when running them.
host_src_dir = build_specified_commit.copy_src_from_docker(
build_data.project_name, tmp_dir)
bisect_repo_manager = repo_manager.BaseRepoManager(
os.path.join(host_src_dir, os.path.basename(repo_path)))
commit_list = bisect_repo_manager.get_commit_list(new_commit, old_commit)
old_idx = len(commit_list) - 1
new_idx = 0
logging.info('Testing against new_commit (%s)', commit_list[new_idx])
if not build_specified_commit.build_fuzzers_from_commit(
commit_list[new_idx], bisect_repo_manager, host_src_dir, build_data):
raise RuntimeError('Failed to build new_commit')
expected_error_code = helper.reproduce_impl(build_data.project_name,
fuzz_target, False, [], [],
test_case_path)
# Check if the error is persistent through the commit range
if old_commit:
logging.info('Testing against old_commit (%s)', commit_list[old_idx])
if not build_specified_commit.build_fuzzers_from_commit(
commit_list[old_idx],
bisect_repo_manager,
host_src_dir,
build_data,
):
raise RuntimeError('Failed to build old_commit')
if expected_error_code == helper.reproduce_impl(build_data.project_name,
fuzz_target, False, [],
[], test_case_path):
return Result(repo_url, commit_list[old_idx])
while old_idx - new_idx > 1:
curr_idx = (old_idx + new_idx) // 2
logging.info('Testing against %s (idx=%d)', commit_list[curr_idx],
curr_idx)
build_specified_commit.build_fuzzers_from_commit(commit_list[curr_idx],
bisect_repo_manager,
host_src_dir, build_data)
error_code = helper.reproduce_impl(build_data.project_name, fuzz_target,
False, [], [], test_case_path)
if expected_error_code == error_code:
new_idx = curr_idx
else:
old_idx = curr_idx
return Result(repo_url, commit_list[new_idx])
def _check_for_crash(project_name, fuzz_target, test_case_path):
"""Check for crash."""
def docker_run(args):
command = ['docker', 'run', '--rm', '--privileged']
if sys.stdin.isatty():
command.append('-i')
return utils.execute(command + args)
logging.info('Checking for crash')
out, err, return_code = helper.reproduce_impl(project_name,
fuzz_target,
False, [], [],
test_case_path,
runner=docker_run,
err_result=(None, None,
None))
if return_code is None:
return None
logging.info('stdout =\n%s', out)
logging.info('stderr =\n%s', err)
# pylint: disable=unsupported-membership-test
has_start_marker = any(marker in out or marker in err
for marker in START_MARKERS)
has_end_marker = any(marker in out or marker in err
for marker in END_MARKERS)
if not has_start_marker or not has_end_marker:
return None
return _get_dedup_token(out + err)
def bisect(commit_old, commit_new, testcase, fuzz_target, build_data):
"""From a commit range, this function caluclates which introduced a
specific error from a fuzz testcase.
Args:
commit_old: The oldest commit in the error regression range
commit_new: The newest commit in the error regression range
testcase: The file path of the test case that triggers the error
fuzz_target: The name of the fuzzer to be tested
build_data: a class holding all of the input parameters for bisection
Returns:
The commit SHA that introduced the error or None
"""
local_store_path = tempfile.mkdtemp()
repo_url = build_specified_commit.infer_main_repo(build_data.project_name,
local_store_path,
commit_old)
bisect_repo_manager = repo_manager.RepoManager(repo_url, local_store_path)
commit_list = bisect_repo_manager.get_commit_list(commit_old, commit_new)
build_specified_commit.build_fuzzer_from_commit(
build_data.project_name, commit_list[0], bisect_repo_manager.repo_dir,
build_data.engine, build_data.sanitizer, build_data.architecture,
bisect_repo_manager)
error_code = helper.reproduce_impl(build_data.project_name, fuzz_target,
False, [], [], testcase)
old_idx = len(commit_list) - 1
new_idx = 0
if len(commit_list) == 1:
if not error_code:
return None
return commit_list[0]
while old_idx - new_idx != 1:
curr_idx = (old_idx + new_idx) // 2
build_specified_commit.build_fuzzer_from_commit(
build_data.project_name, commit_list[curr_idx],
bisect_repo_manager.repo_dir, build_data.engine, build_data.sanitizer,
build_data.architecture, bisect_repo_manager)
error_exists = (
helper.reproduce_impl(build_data.project_name, fuzz_target, False, [],
[], testcase) == error_code)
if error_exists == error_code:
new_idx = curr_idx
else:
old_idx = curr_idx
return commit_list[new_idx]
def test_build_fuzzers_from_commit(self):
"""Tests if the fuzzers can build at a proper commit.
This is done by using a known regression range for a specific test case.
The old commit should show the error when its fuzzers run and the new one
should not.
"""
project_name = 'yara'
old_commit = 'f79be4f2330f4b89ea2f42e1c44ca998c59a0c0f'
new_commit = 'f50a39051ea8c7f10d6d8db9656658b49601caef'
fuzzer = 'rules_fuzzer'
test_data = 'infra/yara_test_data'
build_specified_commit.build_fuzzer_from_commit(
project_name, old_commit, 'tmp', sanitizer='address')
old_error_code = helper.reproduce_impl(project_name, fuzzer, False, [], [],
test_data)
build_specified_commit.build_fuzzer_from_commit(
project_name, new_commit, 'tmp', sanitizer='address')
new_error_code = helper.reproduce_impl(project_name, fuzzer, False, [], [],
test_data)
self.assertNotEqual(new_error_code, old_error_code)
def bisect(old_commit, new_commit, test_case_path, fuzz_target, build_data): # pylint: disable=too-many-locals
"""From a commit range, this function caluclates which introduced a
specific error from a fuzz test_case_path.
Args:
old_commit: The oldest commit in the error regression range.
new_commit: The newest commit in the error regression range.
test_case_path: The file path of the test case that triggers the error
fuzz_target: The name of the fuzzer to be tested.
build_data: a class holding all of the input parameters for bisection.
Returns:
The commit SHA that introduced the error or None.
Raises:
ValueError: when a repo url can't be determine from the project.
"""
with tempfile.TemporaryDirectory() as tmp_dir:
repo_url, repo_path = build_specified_commit.detect_main_repo(
build_data.project_name, commit=new_commit)
if not repo_url or not repo_path:
raise ValueError('Main git repo can not be determined.')
host_src_dir = build_specified_commit.copy_src_from_docker(
build_data.project_name, tmp_dir)
bisect_repo_manager = repo_manager.RepoManager(
repo_url, host_src_dir, repo_name=os.path.basename(repo_path))
commit_list = bisect_repo_manager.get_commit_list(new_commit, old_commit)
old_idx = len(commit_list) - 1
new_idx = 0
logging.info('Testing against new_commit (%s)', commit_list[new_idx])
if not build_specified_commit.build_fuzzers_from_commit(
commit_list[new_idx], bisect_repo_manager, host_src_dir, build_data):
raise RuntimeError('Failed to build new_commit')
expected_error_code = helper.reproduce_impl(build_data.project_name,
fuzz_target, False, [], [],
test_case_path)
# Check if the error is persistent through the commit range
if old_commit:
logging.info('Testing against old_commit (%s)', commit_list[old_idx])
if not build_specified_commit.build_fuzzers_from_commit(
commit_list[old_idx],
bisect_repo_manager,
host_src_dir,
build_data,
):
raise RuntimeError('Failed to build old_commit')
if expected_error_code == helper.reproduce_impl(build_data.project_name,
fuzz_target, False, [],
[], test_case_path):
return commit_list[old_idx]
while old_idx - new_idx > 1:
curr_idx = (old_idx + new_idx) // 2
logging.info('Testing against %s (idx=%d)', commit_list[curr_idx],
curr_idx)
build_specified_commit.build_fuzzers_from_commit(commit_list[curr_idx],
bisect_repo_manager,
host_src_dir, build_data)
error_code = helper.reproduce_impl(build_data.project_name, fuzz_target,
False, [], [], test_case_path)
if expected_error_code == error_code:
new_idx = curr_idx
else:
old_idx = curr_idx
return commit_list[new_idx]
