def test_build_fuzzers_from_commit(self):
"""Tests if the fuzzers can build at a proper commit.
This is done by using a known regression range for a specific test case.
The old commit should show the error when its fuzzers run and the new one
should not.
"""
test_data = os.path.join(TEST_DIR_PATH, 'testcases', 'yara_test_data')
with tempfile.TemporaryDirectory() as tmp_dir:
project_name = 'yara'
old_commit = 'f79be4f2330f4b89ea2f42e1c44ca998c59a0c0f'
new_commit = 'f50a39051ea8c7f10d6d8db9656658b49601caef'
fuzzer = 'rules_fuzzer'
yara_repo_manager = repo_manager.RepoManager(
'https://github.com/VirusTotal/yara.git',
tmp_dir,
repo_name='yara')
build_data = build_specified_commit.BuildData()
build_data.sanitizer = 'address'
build_data.architecture = 'x86_64'
build_data.engine = 'libfuzzer'
build_data.project_name = 'yara'
build_specified_commit.build_fuzzers_from_commit(
old_commit, yara_repo_manager, build_data)
old_error_code = helper.reproduce_impl(project_name, fuzzer, False,
[], [], test_data)
build_specified_commit.build_fuzzers_from_commit(
new_commit, yara_repo_manager, build_data)
new_error_code = helper.reproduce_impl(project_name, fuzzer, False,
[], [], test_data)
self.assertNotEqual(new_error_code, old_error_code)
def test_build_fuzzers_from_commit(self):
"""Tests if the fuzzers can build at a specified commit.
This is done by using a known regression range for a specific test case.
The old commit should show the error when its fuzzers run and the new one
should not.
"""
with tempfile.TemporaryDirectory() as tmp_dir:
test_case = test_repos.TEST_REPOS[1]
self.assertTrue(helper.build_image_impl(test_case.project_name))
host_src_dir = build_specified_commit.copy_src_from_docker(
test_case.project_name, tmp_dir)
test_repo_manager = repo_manager.clone_repo_and_get_manager(
test_case.git_url, host_src_dir, test_case.oss_repo_name)
build_data = build_specified_commit.BuildData(
sanitizer='address',
architecture='x86_64',
engine='libfuzzer',
project_name=test_case.project_name)
build_specified_commit.build_fuzzers_from_commit(test_case.old_commit,
test_repo_manager,
host_src_dir, build_data)
old_error_code = helper.reproduce_impl(test_case.project_name,
test_case.fuzz_target, False, [],
[], test_case.test_case_path)
build_specified_commit.build_fuzzers_from_commit(test_case.new_commit,
test_repo_manager,
host_src_dir, build_data)
new_error_code = helper.reproduce_impl(test_case.project_name,
test_case.fuzz_target, False, [],
[], test_case.test_case_path)
self.assertNotEqual(new_error_code, old_error_code)
def _bisect(old_commit, new_commit, test_case_path, fuzz_target, build_data): # pylint: disable=too-many-locals
"""Perform the bisect."""
with tempfile.TemporaryDirectory() as tmp_dir:
repo_url, repo_path = build_specified_commit.detect_main_repo(
build_data.project_name, commit=new_commit)
if not repo_url or not repo_path:
raise ValueError('Main git repo can not be determined.')
# Copy /src from the built Docker container to ensure all dependencies
# exist. This will be mounted when running them.
host_src_dir = build_specified_commit.copy_src_from_docker(
build_data.project_name, tmp_dir)
bisect_repo_manager = repo_manager.BaseRepoManager(
os.path.join(host_src_dir, os.path.basename(repo_path)))
commit_list = bisect_repo_manager.get_commit_list(new_commit, old_commit)
old_idx = len(commit_list) - 1
new_idx = 0
logging.info('Testing against new_commit (%s)', commit_list[new_idx])
if not build_specified_commit.build_fuzzers_from_commit(
commit_list[new_idx], bisect_repo_manager, host_src_dir, build_data):
raise RuntimeError('Failed to build new_commit')
expected_error_code = helper.reproduce_impl(build_data.project_name,
fuzz_target, False, [], [],
test_case_path)
# Check if the error is persistent through the commit range
if old_commit:
logging.info('Testing against old_commit (%s)', commit_list[old_idx])
if not build_specified_commit.build_fuzzers_from_commit(
commit_list[old_idx],
bisect_repo_manager,
host_src_dir,
build_data,
):
raise RuntimeError('Failed to build old_commit')
if expected_error_code == helper.reproduce_impl(build_data.project_name,
fuzz_target, False, [],
[], test_case_path):
return Result(repo_url, commit_list[old_idx])
while old_idx - new_idx > 1:
curr_idx = (old_idx + new_idx) // 2
logging.info('Testing against %s (idx=%d)', commit_list[curr_idx],
curr_idx)
build_specified_commit.build_fuzzers_from_commit(commit_list[curr_idx],
bisect_repo_manager,
host_src_dir, build_data)
error_code = helper.reproduce_impl(build_data.project_name, fuzz_target,
False, [], [], test_case_path)
if expected_error_code == error_code:
new_idx = curr_idx
else:
old_idx = curr_idx
return Result(repo_url, commit_list[new_idx])
def bisect(old_commit, new_commit, test_case_path, fuzz_target, build_data):
"""From a commit range, this function caluclates which introduced a
specific error from a fuzz test_case_path.
Args:
old_commit: The oldest commit in the error regression range.
new_commit: The newest commit in the error regression range.
test_case_path: The file path of the test case that triggers the error
fuzz_target: The name of the fuzzer to be tested.
build_data: a class holding all of the input parameters for bisection.
Returns:
The commit SHA that introduced the error or None.
Raises:
ValueError: when a repo url can't be determine from the project.
"""
with tempfile.TemporaryDirectory() as tmp_dir:
repo_url, repo_name = build_specified_commit.detect_main_repo(
build_data.project_name, commit=old_commit)
if not repo_url or not repo_name:
raise ValueError('Main git repo can not be determined.')
bisect_repo_manager = repo_manager.RepoManager(repo_url,
tmp_dir,
repo_name=repo_name)
commit_list = bisect_repo_manager.get_commit_list(
old_commit, new_commit)
old_idx = len(commit_list) - 1
new_idx = 0
build_specified_commit.build_fuzzers_from_commit(
commit_list[new_idx], bisect_repo_manager, build_data)
expected_error_code = helper.reproduce_impl(build_data.project_name,
fuzz_target, False, [], [],
test_case_path)
# Check if the error is persistent through the commit range
build_specified_commit.build_fuzzers_from_commit(
commit_list[old_idx],
bisect_repo_manager,
build_data,
)
if expected_error_code == helper.reproduce_impl(
build_data.project_name, fuzz_target, False, [], [],
test_case_path):
return commit_list[old_idx]
while old_idx - new_idx > 1:
curr_idx = (old_idx + new_idx) // 2
build_specified_commit.build_fuzzers_from_commit(
commit_list[curr_idx], bisect_repo_manager, build_data)
error_code = helper.reproduce_impl(build_data.project_name,
fuzz_target, False, [], [],
test_case_path)
if expected_error_code == error_code:
new_idx = curr_idx
else:
old_idx = curr_idx
return commit_list[new_idx]
def build_fuzzers(args):
"""Builds all of the fuzzers for a specific OSS-Fuzz project.
Returns:
True on success False on failure.
"""
# TODO: Fix return value bubble to actually handle errors.
with tempfile.TemporaryDirectory() as tmp_dir:
inferred_url, repo_name = build_specified_commit.detect_main_repo(
args.project_name, repo_name=args.repo_name)
build_repo_manager = repo_manager.RepoManager(inferred_url,
tmp_dir,
repo_name=repo_name)
build_data = build_specified_commit.BuildData()
build_data.project_name = args.project_name
build_data.sanitizer = 'address'
build_data.engine = 'libfuzzer'
build_data.architecture = 'x86_64'
return build_specified_commit.build_fuzzers_from_commit(
args.commit_sha, build_repo_manager, build_data) == 0
def bisect(old_commit, new_commit, test_case_path, fuzz_target, build_data): # pylint: disable=too-many-locals
"""From a commit range, this function caluclates which introduced a
specific error from a fuzz test_case_path.
Args:
old_commit: The oldest commit in the error regression range.
new_commit: The newest commit in the error regression range.
test_case_path: The file path of the test case that triggers the error
fuzz_target: The name of the fuzzer to be tested.
build_data: a class holding all of the input parameters for bisection.
Returns:
The commit SHA that introduced the error or None.
Raises:
ValueError: when a repo url can't be determine from the project.
"""
with tempfile.TemporaryDirectory() as tmp_dir:
repo_url, repo_path = build_specified_commit.detect_main_repo(
build_data.project_name, commit=new_commit)
if not repo_url or not repo_path:
raise ValueError('Main git repo can not be determined.')
host_src_dir = build_specified_commit.copy_src_from_docker(
build_data.project_name, tmp_dir)
bisect_repo_manager = repo_manager.RepoManager(
repo_url, host_src_dir, repo_name=os.path.basename(repo_path))
commit_list = bisect_repo_manager.get_commit_list(new_commit, old_commit)
old_idx = len(commit_list) - 1
new_idx = 0
logging.info('Testing against new_commit (%s)', commit_list[new_idx])
if not build_specified_commit.build_fuzzers_from_commit(
commit_list[new_idx], bisect_repo_manager, host_src_dir, build_data):
raise RuntimeError('Failed to build new_commit')
expected_error_code = helper.reproduce_impl(build_data.project_name,
fuzz_target, False, [], [],
test_case_path)
# Check if the error is persistent through the commit range
if old_commit:
logging.info('Testing against old_commit (%s)', commit_list[old_idx])
if not build_specified_commit.build_fuzzers_from_commit(
commit_list[old_idx],
bisect_repo_manager,
host_src_dir,
build_data,
):
raise RuntimeError('Failed to build old_commit')
if expected_error_code == helper.reproduce_impl(build_data.project_name,
fuzz_target, False, [],
[], test_case_path):
return commit_list[old_idx]
while old_idx - new_idx > 1:
curr_idx = (old_idx + new_idx) // 2
logging.info('Testing against %s (idx=%d)', commit_list[curr_idx],
curr_idx)
build_specified_commit.build_fuzzers_from_commit(commit_list[curr_idx],
bisect_repo_manager,
host_src_dir, build_data)
error_code = helper.reproduce_impl(build_data.project_name, fuzz_target,
False, [], [], test_case_path)
if expected_error_code == error_code:
new_idx = curr_idx
else:
old_idx = curr_idx
return commit_list[new_idx]
def _bisect(bisect_type, old_commit, new_commit, test_case_path, fuzz_target,
build_data):
"""Perform the bisect."""
# pylint: disable=too-many-branches
base_builder_repo = build_specified_commit.load_base_builder_repo()
with tempfile.TemporaryDirectory() as tmp_dir:
repo_url, repo_path = build_specified_commit.detect_main_repo(
build_data.project_name, commit=new_commit)
if not repo_url or not repo_path:
raise ValueError('Main git repo can not be determined.')
if old_commit == new_commit:
raise BisectError('old_commit is the same as new_commit', repo_url)
# Copy /src from the built Docker container to ensure all dependencies
# exist. This will be mounted when running them.
host_src_dir = build_specified_commit.copy_src_from_docker(
build_data.project_name, tmp_dir)
bisect_repo_manager = repo_manager.BaseRepoManager(
os.path.join(host_src_dir, os.path.basename(repo_path)))
commit_list = bisect_repo_manager.get_commit_list(
new_commit, old_commit)
old_idx = len(commit_list) - 1
new_idx = 0
logging.info('Testing against new_commit (%s)', commit_list[new_idx])
if not build_specified_commit.build_fuzzers_from_commit(
commit_list[new_idx],
bisect_repo_manager,
host_src_dir,
build_data,
base_builder_repo=base_builder_repo):
raise BisectError('Failed to build new_commit', repo_url)
if bisect_type == 'fixed':
should_crash = False
elif bisect_type == 'regressed':
should_crash = True
else:
raise BisectError('Invalid bisect type ' + bisect_type, repo_url)
expected_error = _check_for_crash(build_data.project_name, fuzz_target,
test_case_path)
logging.info('new_commit result = %s', expected_error)
if not should_crash and expected_error:
logging.warning('new_commit crashed but not shouldn\'t. '
'Continuing to see if stack changes.')
range_valid = False
for _ in range(2):
logging.info('Testing against old_commit (%s)',
commit_list[old_idx])
if not build_specified_commit.build_fuzzers_from_commit(
commit_list[old_idx],
bisect_repo_manager,
host_src_dir,
build_data,
base_builder_repo=base_builder_repo):
raise BisectError('Failed to build old_commit', repo_url)
if _check_for_crash(build_data.project_name, fuzz_target,
test_case_path) == expected_error:
logging.warning(
'old_commit %s had same result as new_commit %s',
old_commit, new_commit)
# Try again on an slightly older commit.
old_commit = bisect_repo_manager.get_parent(old_commit, 64)
if not old_commit:
break
commit_list = bisect_repo_manager.get_commit_list(
new_commit, old_commit)
old_idx = len(commit_list) - 1
continue
range_valid = True
break
if not range_valid:
raise BisectError('old_commit had same result as new_commit',
repo_url)
while old_idx - new_idx > 1:
curr_idx = (old_idx + new_idx) // 2
logging.info('Testing against %s (idx=%d)', commit_list[curr_idx],
curr_idx)
if not build_specified_commit.build_fuzzers_from_commit(
commit_list[curr_idx],
bisect_repo_manager,
host_src_dir,
build_data,
base_builder_repo=base_builder_repo):
# Treat build failures as if we couldn't repo.
# TODO(ochang): retry nearby commits?
old_idx = curr_idx
continue
current_error = _check_for_crash(build_data.project_name,
fuzz_target, test_case_path)
logging.info('Current result = %s', current_error)
if expected_error == current_error:
new_idx = curr_idx
else:
old_idx = curr_idx
return Result(repo_url, commit_list[new_idx])
def _bisect(bisect_type, old_commit, new_commit, test_case_path, fuzz_target,
build_data):
"""Perform the bisect."""
# pylint: disable=too-many-branches
base_builder_repo = _load_base_builder_repo()
with tempfile.TemporaryDirectory() as tmp_dir:
repo_url, repo_path = build_specified_commit.detect_main_repo(
build_data.project_name, commit=new_commit)
if not repo_url or not repo_path:
raise ValueError('Main git repo can not be determined.')
# Copy /src from the built Docker container to ensure all dependencies
# exist. This will be mounted when running them.
host_src_dir = build_specified_commit.copy_src_from_docker(
build_data.project_name, tmp_dir)
bisect_repo_manager = repo_manager.BaseRepoManager(
os.path.join(host_src_dir, os.path.basename(repo_path)))
commit_list = bisect_repo_manager.get_commit_list(
new_commit, old_commit)
old_idx = len(commit_list) - 1
new_idx = 0
logging.info('Testing against new_commit (%s)', commit_list[new_idx])
if not build_specified_commit.build_fuzzers_from_commit(
commit_list[new_idx],
bisect_repo_manager,
host_src_dir,
build_data,
base_builder_repo=base_builder_repo):
raise RuntimeError('Failed to build new_commit')
if bisect_type == 'fixed':
expected_error = False
elif bisect_type == 'regressed':
expected_error = True
else:
raise ValueError('Invalid bisect type ' + bisect_type)
if _check_for_crash(build_data.project_name, fuzz_target,
test_case_path) != expected_error:
raise RuntimeError('new_commit didn\'t have expected result.')
# Check if the error is persistent through the commit range
if old_commit:
logging.info('Testing against old_commit (%s)',
commit_list[old_idx])
if not build_specified_commit.build_fuzzers_from_commit(
commit_list[old_idx],
bisect_repo_manager,
host_src_dir,
build_data,
base_builder_repo=base_builder_repo):
raise RuntimeError('Failed to build old_commit')
if _check_for_crash(build_data.project_name, fuzz_target,
test_case_path) == expected_error:
raise RuntimeError('old_commit had same result as new_commit')
while old_idx - new_idx > 1:
curr_idx = (old_idx + new_idx) // 2
logging.info('Testing against %s (idx=%d)', commit_list[curr_idx],
curr_idx)
if not build_specified_commit.build_fuzzers_from_commit(
commit_list[curr_idx],
bisect_repo_manager,
host_src_dir,
build_data,
base_builder_repo=base_builder_repo):
# Treat build failures as if we couldn't repo.
# TODO(ochang): retry nearby commits?
old_idx = curr_idx
continue
is_error = _check_for_crash(build_data.project_name, fuzz_target,
test_case_path)
if expected_error == is_error:
new_idx = curr_idx
else:
old_idx = curr_idx
return Result(repo_url, commit_list[new_idx])
