def test_coverage_report(self, _):
"""Tests generation of coverage reports end-to-end, from building to
generation."""
with test_helpers.docker_temp_dir() as temp_dir:
shared = os.path.join(temp_dir, 'shared')
os.mkdir(shared)
copy_command = ('cp -r /opt/code_coverage /shared && '
'cp $(which llvm-profdata) /shared && '
'cp $(which llvm-cov) /shared')
assert helper.docker_run([
'-v', f'{shared}:/shared', 'gcr.io/oss-fuzz-base/base-runner',
'bash', '-c', copy_command
])
os.environ['CODE_COVERAGE_SRC'] = os.path.join(
shared, 'code_coverage')
os.environ['PATH'] += os.pathsep + shared
# Do coverage build.
build_config = test_helpers.create_build_config(
oss_fuzz_project_name=EXAMPLE_PROJECT,
project_repo_name='oss-fuzz',
workspace=temp_dir,
commit_sha='0b95fe1039ed7c38fea1f97078316bfc1030c523',
base_commit='da0746452433dc18bae699e355a9821285d863c8',
sanitizer=self.SANITIZER,
is_github=True,
# Needed for test not to fail because of permissions issues.
bad_build_check=False)
self.assertTrue(build_fuzzers.build_fuzzers(build_config))
# TODO(metzman): Get rid of this here and make 'compile' do this.
chmod_command = ('chmod -R +r /out && '
'find /out -type d -exec chmod +x {} +')
assert helper.docker_run([
'-v', f'{os.path.join(temp_dir, "build-out")}:/out',
'gcr.io/oss-fuzz-base/base-builder', 'bash', '-c',
chmod_command
])
# Generate report.
run_config = test_helpers.create_run_config(
fuzz_seconds=FUZZ_SECONDS,
workspace=temp_dir,
sanitizer=self.SANITIZER,
run_fuzzers_mode='coverage',
is_github=True)
result = run_fuzzers.run_fuzzers(run_config)
self.assertEqual(result, run_fuzzers.RunFuzzersResult.NO_BUG_FOUND)
expected_summary_path = os.path.join(
TEST_DATA_PATH, 'example_coverage_report_summary.json')
with open(expected_summary_path) as file_handle:
expected_summary = json.loads(file_handle.read())
actual_summary_path = os.path.join(temp_dir, 'cifuzz-coverage',
'report', 'linux',
'summary.json')
with open(actual_summary_path) as file_handle:
actual_summary = json.loads(file_handle.read())
self.assertEqual(expected_summary, actual_summary)
def handle_msan_postbuild(self, container):
"""Post-build step for MSAN builds. Patches the build to use MSAN
libraries."""
helper.docker_run([
'--volumes-from', container, '-e', f'WORK={self.work_dir}',
docker.MSAN_LIBS_BUILDER_TAG, 'patch_build.py', '/out'
])
def handle_msan_prebuild(self, container):
"""Pre-build step for MSAN builds. Copies MSAN libs to |msan_libs_dir| and
returns docker arguments to use that directory for MSAN libs."""
logging.info('Copying MSAN libs.')
helper.docker_run([
'--volumes-from', container, docker.MSAN_LIBS_BUILDER_TAG, 'bash',
'-c', f'cp -r /msan {self.work_dir}'
])
def handle_msan_postbuild(self, container):
"""Post-build step for MSAN builds. Patches the build to use MSAN
libraries."""
helper.docker_run([
'--volumes-from', container, '-e',
'WORK={work_dir}'.format(work_dir=self.work_dir),
'gcr.io/oss-fuzz-base/base-sanitizer-libs-builder',
'patch_build.py', '/out'
])
def handle_msan_prebuild(self, container):
"""Pre-build step for MSAN builds. Copies MSAN libs to |msan_libs_dir| and
returns docker arguments to use that directory for MSAN libs."""
logging.info('Copying MSAN libs.')
helper.docker_run([
'--volumes-from', container,
'gcr.io/oss-fuzz-base/msan-libs-builder', 'bash', '-c',
'cp -r /msan {work_dir}'.format(work_dir=self.work_dir)
])
def docker_temp_dir():
"""Returns a temporary a directory that is useful for use with docker. On
cleanup this contextmanager uses docker to delete the directory's contents so
that if anything is owned by root it can be deleted (which
tempfile.TemporaryDirectory() cannot do) by non-root users."""
with tempfile.TemporaryDirectory() as temp_dir:
yield temp_dir
helper.docker_run([
'-v', f'{temp_dir}:/temp_dir', '-t', docker.BASE_BUILDER_TAG,
'/bin/bash', '-c', 'rm -rf /temp_dir/*'
])
def test_coverage_report(self, _):
"""Tests generation of coverage reports end-to-end, from building to
generation."""
with tempfile.TemporaryDirectory() as workspace:
try:
# Do coverage build.
build_config = test_helpers.create_build_config(
oss_fuzz_project_name=EXAMPLE_PROJECT,
project_repo_name='oss-fuzz',
workspace=workspace,
commit_sha='0b95fe1039ed7c38fea1f97078316bfc1030c523',
base_commit='da0746452433dc18bae699e355a9821285d863c8',
sanitizer=self.SANITIZER,
is_github=True)
self.assertTrue(build_fuzzers.build_fuzzers(build_config))
# Generate report.
run_config = test_helpers.create_run_config(
fuzz_seconds=FUZZ_SECONDS,
workspace=workspace,
oss_fuzz_project_name=EXAMPLE_PROJECT,
sanitizer=self.SANITIZER,
run_fuzzers_mode='coverage',
is_github=True,
# Set build integration path so it's not internal.
build_integration_path='/')
result = run_fuzzers.run_fuzzers(run_config)
self.assertEqual(result,
run_fuzzers.RunFuzzersResult.NO_BUG_FOUND)
expected_summary_path = os.path.join(
TEST_DATA_PATH, 'example_coverage_report_summary.json')
with open(expected_summary_path) as file_handle:
expected_summary = json.loads(file_handle.read())
actual_summary_path = os.path.join(workspace,
'cifuzz-coverage', 'report',
'linux', 'summary.json')
with open(actual_summary_path) as file_handle:
actual_summary = json.loads(file_handle.read())
self.assertEqual(expected_summary, actual_summary)
finally:
# If we don't do this, there will be an exception when the temporary
# directory is deleted because there are files there that are only
# writeable by root.
if os.listdir(workspace):
helper.docker_run([
'-v', f'{workspace}:/workspace', '-t',
docker.BASE_RUNNER_TAG, '/bin/bash', '-c',
'rm -rf /workspace/*'
])
def check_fuzzer_build(workspace,
sanitizer,
language,
allowed_broken_targets_percentage=None):
"""Checks the integrity of the built fuzzers.
Args:
out_dir: The directory containing the fuzzer binaries.
sanitizer: The sanitizer the fuzzers are built with.
Returns:
True if fuzzers are correct.
"""
if not os.path.exists(workspace.out):
logging.error('Invalid out directory: %s.', workspace.out)
return False
if not os.listdir(workspace.out):
logging.error('No fuzzers found in out directory: %s.', workspace.out)
return False
docker_args, _ = docker.get_base_docker_run_args(workspace, sanitizer,
language)
if allowed_broken_targets_percentage is not None:
docker_args += [
'-e',
('ALLOWED_BROKEN_TARGETS_PERCENTAGE=' +
allowed_broken_targets_percentage)
]
docker_args.extend(['-t', docker.BASE_RUNNER_TAG, 'test_all.py'])
result = helper.docker_run(docker_args)
if not result:
logging.error('Check fuzzer build failed.')
return False
return True
def build_fuzzers(self):
"""Moves the source code we want to fuzz into the project builder and builds
the fuzzers from that source code. Returns True on success."""
docker_args, docker_container = docker.get_base_docker_run_args(
self.workspace, self.config.sanitizer, self.config.language,
self.config.architecture, self.config.docker_in_docker)
if not docker_container:
docker_args.extend(
_get_docker_build_fuzzers_args_not_container(
self.host_repo_path))
build_command = self.ci_system.get_build_command(
self.host_repo_path, self.image_repo_path)
docker_args.extend([
docker.get_project_image_name(self.config.oss_fuzz_project_name),
'/bin/bash',
'-c',
build_command,
])
logging.info('Building with %s sanitizer.', self.config.sanitizer)
# TODO(metzman): Stop using helper.docker_run so we can get rid of
# docker.get_base_docker_run_args and merge its contents into
# docker.get_base_docker_run_command.
if not helper.docker_run(docker_args):
logging.error('Building fuzzers failed.')
return False
return True
def check_fuzzer_build(out_dir):
"""Checks the integrity of the built fuzzers.
Args:
out_dir: The directory containing the fuzzer binaries.
Returns:
True if fuzzers are correct.
"""
if not os.path.exists(out_dir):
logging.error('Invalid out directory: %s.', out_dir)
return False
if not os.listdir(out_dir):
logging.error('No fuzzers found in out directory: %s.', out_dir)
return False
command = [
'--cap-add', 'SYS_PTRACE', '-e', 'FUZZING_ENGINE=' + DEFAULT_ENGINE,
'-e', 'SANITIZER=' + DEFAULT_SANITIZER, '-e',
'ARCHITECTURE=' + DEFAULT_ARCHITECTURE
]
container = utils.get_container_name()
if container:
command += ['-e', 'OUT=' + out_dir, '--volumes-from', container]
else:
command += ['-v', '%s:/out' % out_dir]
command.extend(['-t', 'gcr.io/oss-fuzz-base/base-runner', 'test_all'])
exit_code = helper.docker_run(command)
if exit_code:
logging.error('Check fuzzer build failed.')
return False
return True
def copy_src_from_docker(project_name, host_dir):
"""Copy /src from docker to the host."""
# Copy /src to host.
image_name = 'gcr.io/oss-fuzz/' + project_name
docker_args = [
'-v',
host_dir + ':/out',
image_name,
'cp',
'-r',
'-p',
'/src',
'/out',
]
helper.docker_run(docker_args)
return os.path.join(host_dir, 'src')
def build_fuzzers(self):
"""Moves the source code we want to fuzz into the project builder and builds
the fuzzers from that source code. Returns True on success."""
image_src_path = os.path.dirname(self.image_repo_path)
command = get_common_docker_args(self.sanitizer)
container = utils.get_container_name()
if container:
command.extend(
['-e', 'OUT=' + self.out_dir, '--volumes-from', container])
rm_path = os.path.join(self.image_repo_path, '*')
bash_command = 'rm -rf {0} && cp -r {1} {2} && compile'.format(
rm_path, self.host_repo_path, image_src_path)
else:
# TODO(metzman): Figure out if we can eliminate this branch.
command.extend([
'-e', 'OUT=' + '/out', '-v',
'%s:%s' % (self.host_repo_path, self.image_repo_path), '-v',
'%s:%s' % (self.out_dir, '/out')
])
bash_command = 'compile'
command.extend([
'gcr.io/oss-fuzz/' + self.project_name,
'/bin/bash',
'-c',
])
command.append(bash_command)
logging.info('Building with %s sanitizer.', self.sanitizer)
if helper.docker_run(command):
# docker_run returns nonzero on failure.
logging.error('Building fuzzers failed.')
return False
return True
def run_coverage_command(workspace, config):
"""Runs the coverage command in base-runner to generate a coverage report."""
docker_args, _ = docker.get_base_docker_run_args(workspace,
config.sanitizer,
config.language)
docker_args += [
'-e', 'COVERAGE_EXTRA_ARGS=', '-e', 'HTTP_PORT=', '-t',
docker.BASE_RUNNER_TAG, 'coverage'
]
return helper.docker_run(docker_args)
def check_fuzzer_build(out_dir, sanitizer='address'):
"""Checks the integrity of the built fuzzers.
Args:
out_dir: The directory containing the fuzzer binaries.
sanitizer: The sanitizer the fuzzers are built with.
Returns:
True if fuzzers are correct.
"""
if not os.path.exists(out_dir):
logging.error('Invalid out directory: %s.', out_dir)
return False
if not os.listdir(out_dir):
logging.error('No fuzzers found in out directory: %s.', out_dir)
return False
command = [
'--cap-add',
'SYS_PTRACE',
'-e',
'FUZZING_ENGINE=' + DEFAULT_ENGINE,
'-e',
'SANITIZER=' + sanitizer,
'-e',
'ARCHITECTURE=' + DEFAULT_ARCHITECTURE,
'-e',
'CIFUZZ=True',
'-e',
'FUZZING_LANGUAGE=c++', # FIXME: Add proper support.
]
# Set ALLOWED_BROKEN_TARGETS_PERCENTAGE in docker if specified by user.
allowed_broken_targets_percentage = os.getenv(
'ALLOWED_BROKEN_TARGETS_PERCENTAGE')
if allowed_broken_targets_percentage is not None:
command += [
'-e',
('ALLOWED_BROKEN_TARGETS_PERCENTAGE=' +
allowed_broken_targets_percentage)
]
container = utils.get_container_name()
if container:
command += ['-e', 'OUT=' + out_dir, '--volumes-from', container]
else:
command += ['-v', '%s:/out' % out_dir]
command.extend(['-t', 'gcr.io/oss-fuzz-base/base-runner', 'test_all'])
exit_code = helper.docker_run(command)
if exit_code:
logging.error('Check fuzzer build failed.')
return False
return True
def copy_src_from_docker(project_name, host_dir):
"""Copy /src from docker to the host."""
# Copy /src to host.
image_name = 'gcr.io/oss-fuzz/' + project_name
docker_args = [
'-v',
host_dir + ':/out',
image_name,
'rsync',
'-az',
'--delete',
'/src',
'/out',
]
helper.docker_run(docker_args)
# Submodules can have gitdir entries which point to absolute paths. Make them
# relative, as otherwise we can't do operations on the checkout on the host.
src_dir = os.path.join(host_dir, 'src')
_make_gitdirs_relative(src_dir)
return src_dir
def copy_src_from_docker(project_name, host_dir):
"""Copy /src from docker to the host."""
# Copy /src to host.
image_name = 'gcr.io/oss-fuzz/' + project_name
src_dir = os.path.join(host_dir, 'src')
if os.path.exists(src_dir):
shutil.rmtree(src_dir, ignore_errors=True)
docker_args = [
'-v',
host_dir + ':/out',
image_name,
'cp',
'-r',
'-p',
'/src',
'/out',
]
helper.docker_run(docker_args)
# Submodules can have gitdir entries which point to absolute paths. Make them
# relative, as otherwise we can't do operations on the checkout on the host.
_make_gitdirs_relative(src_dir)
return src_dir
def _copy_repo_from_image(self, image_repo_path):
self._make_repo_storage_dir()
repo_name = os.path.basename(image_repo_path)
host_repo_path = os.path.join(self._repo_dir, repo_name)
bash_command = f'cp -r {image_repo_path} {host_repo_path}'
docker_args, _ = docker.get_base_docker_run_args(
self.workspace, self.config.sanitizer, self.config.language,
self.config.docker_in_docker)
docker_args.extend([
docker.get_project_image_name(self.config.oss_fuzz_project_name),
'/bin/bash', '-c', bash_command
])
if not helper.docker_run(docker_args):
raise RuntimeError('Failed to copy repo.')
return repo_manager.RepoManager(host_repo_path)
def check_fuzzer_build(out_dir,
sanitizer,
language,
allowed_broken_targets_percentage=None):
"""Checks the integrity of the built fuzzers.
Args:
out_dir: The directory containing the fuzzer binaries.
sanitizer: The sanitizer the fuzzers are built with.
Returns:
True if fuzzers are correct.
"""
if not os.path.exists(out_dir):
logging.error('Invalid out directory: %s.', out_dir)
return False
if not os.listdir(out_dir):
logging.error('No fuzzers found in out directory: %s.', out_dir)
return False
command = get_common_docker_args(sanitizer, language)
if allowed_broken_targets_percentage is not None:
command += [
'-e',
('ALLOWED_BROKEN_TARGETS_PERCENTAGE=' +
allowed_broken_targets_percentage)
]
container = utils.get_container_name()
if container:
command += ['-e', 'OUT=' + out_dir, '--volumes-from', container]
else:
command += ['-v', '%s:/out' % out_dir]
command.extend(['-t', 'gcr.io/oss-fuzz-base/base-runner', 'test_all.py'])
exit_code = helper.docker_run(command)
logging.info('check fuzzer build exit code: %d', exit_code)
if exit_code:
logging.error('Check fuzzer build failed.')
return False
return True
def build_fuzzers(self):
"""Moves the source code we want to fuzz into the project builder and builds
the fuzzers from that source code. Returns True on success."""
docker_args = get_common_docker_args(self.config.sanitizer,
self.config.language)
container = utils.get_container_name()
if container:
docker_args.extend(
_get_docker_build_fuzzers_args_container(
self.out_dir, container))
else:
docker_args.extend(
_get_docker_build_fuzzers_args_not_container(
self.out_dir, self.host_repo_path))
if self.config.sanitizer == 'memory':
docker_args.extend(
_get_docker_build_fuzzers_args_msan(self.work_dir))
self.handle_msan_prebuild(container)
docker_args.extend([
'gcr.io/oss-fuzz/' + self.config.project_name,
'/bin/bash',
'-c',
])
rm_path = os.path.join(self.image_repo_path, '*')
image_src_path = os.path.dirname(self.image_repo_path)
bash_command = 'rm -rf {0} && cp -r {1} {2} && compile'.format(
rm_path, self.host_repo_path, image_src_path)
docker_args.append(bash_command)
logging.info('Building with %s sanitizer.', self.config.sanitizer)
if helper.docker_run(docker_args):
# docker_run returns nonzero on failure.
logging.error('Building fuzzers failed.')
return False
if self.config.sanitizer == 'memory':
self.handle_msan_postbuild(container)
return True
def check_fuzzer_build(workspace,
sanitizer,
language,
allowed_broken_targets_percentage=None):
"""Checks the integrity of the built fuzzers.
Args:
workspace: The workspace used by CIFuzz.
sanitizer: The sanitizer the fuzzers are built with.
language: The programming language the fuzzers are written in.
allowed_broken_targets_percentage (optional): A custom percentage of broken
targets to allow.
Returns:
True if fuzzers pass OSS-Fuzz's build check.
"""
if not os.path.exists(workspace.out):
logging.error('Invalid out directory: %s.', workspace.out)
return False
if not os.listdir(workspace.out):
logging.error('No fuzzers found in out directory: %s.', workspace.out)
return False
docker_args, _ = docker.get_base_docker_run_args(workspace, sanitizer,
language)
if allowed_broken_targets_percentage is not None:
docker_args += [
'-e',
('ALLOWED_BROKEN_TARGETS_PERCENTAGE=' +
allowed_broken_targets_percentage)
]
docker_args.extend(['-t', docker.BASE_RUNNER_TAG, 'test_all.py'])
result = helper.docker_run(docker_args)
if not result:
logging.error('Check fuzzer build failed.')
return False
return True
def build_fuzzers(self):
"""Moves the source code we want to fuzz into the project builder and builds
the fuzzers from that source code. Returns True on success."""
docker_args, docker_container = docker.get_base_docker_run_args(
self.workspace, self.config.sanitizer, self.config.language)
if not docker_container:
docker_args.extend(
_get_docker_build_fuzzers_args_not_container(
self.host_repo_path))
if self.config.sanitizer == 'memory':
docker_args.extend(
_get_docker_build_fuzzers_args_msan(self.workspace.work))
self.handle_msan_prebuild(docker_container)
docker_args.extend([
docker.get_project_image_name(self.config.project_name),
'/bin/bash',
'-c',
])
rm_path = os.path.join(self.image_repo_path, '*')
image_src_path = os.path.dirname(self.image_repo_path)
bash_command = (f'rm -rf {rm_path} && cp -r {self.host_repo_path} '
f'{image_src_path} && compile')
docker_args.append(bash_command)
logging.info('Building with %s sanitizer.', self.config.sanitizer)
# TODO(metzman): Stop using helper.docker_run so we can get rid of
# docker.get_base_docker_run_args and merge its contents into
# docker.get_base_docker_run_command.
if not helper.docker_run(docker_args):
logging.error('Building fuzzers failed.')
return False
if self.config.sanitizer == 'memory':
self.handle_msan_postbuild(docker_container)
return True
def build_fuzzers(project_name,
project_repo_name,
workspace,
pr_ref=None,
commit_sha=None,
sanitizer='address'):
"""Builds all of the fuzzers for a specific OSS-Fuzz project.
Args:
project_name: The name of the OSS-Fuzz project being built.
project_repo_name: The name of the projects repo.
workspace: The location in a shared volume to store a git repo and build
artifacts.
pr_ref: The pull request reference to be built.
commit_sha: The commit sha for the project to be built at.
sanitizer: The sanitizer the fuzzers should be built with.
Returns:
True if build succeeded or False on failure.
"""
# Validate inputs.
assert pr_ref or commit_sha
if not os.path.exists(workspace):
logging.error('Invalid workspace: %s.', workspace)
return False
logging.info("Using %s sanitizer.", sanitizer)
out_dir = os.path.join(workspace, 'out')
os.makedirs(out_dir, exist_ok=True)
# Build Fuzzers using docker run.
inferred_url, project_builder_repo_path = (
build_specified_commit.detect_main_repo(project_name,
repo_name=project_repo_name))
if not inferred_url or not project_builder_repo_path:
logging.error('Could not detect repo from project %s.', project_name)
return False
project_repo_name = os.path.basename(project_builder_repo_path)
src_in_project_builder = os.path.dirname(project_builder_repo_path)
manual_src_path = os.getenv('MANUAL_SRC_PATH')
if manual_src_path:
if not os.path.exists(manual_src_path):
logging.error(
'MANUAL_SRC_PATH: %s does not exist. '
'Are you mounting it correctly?', manual_src_path)
return False
# This is the path taken outside of GitHub actions.
git_workspace = os.path.dirname(manual_src_path)
else:
git_workspace = os.path.join(workspace, 'storage')
os.makedirs(git_workspace, exist_ok=True)
# Checkout projects repo in the shared volume.
build_repo_manager = repo_manager.RepoManager(inferred_url,
git_workspace,
repo_name=project_repo_name)
if not manual_src_path:
checkout_specified_commit(build_repo_manager, pr_ref, commit_sha)
command = [
'--cap-add',
'SYS_PTRACE',
'-e',
'FUZZING_ENGINE=' + DEFAULT_ENGINE,
'-e',
'SANITIZER=' + sanitizer,
'-e',
'ARCHITECTURE=' + DEFAULT_ARCHITECTURE,
'-e',
'FUZZING_LANGUAGE=c++', # FIXME: Add proper support.
]
container = utils.get_container_name()
if container:
command += ['-e', 'OUT=' + out_dir, '--volumes-from', container]
bash_command = 'rm -rf {0} && cp -r {1} {2} && compile'.format(
os.path.join(src_in_project_builder, project_repo_name, '*'),
os.path.join(git_workspace, project_repo_name),
src_in_project_builder)
else:
command += [
'-e', 'OUT=' + '/out', '-v',
'%s:%s' %
(os.path.join(git_workspace, project_repo_name),
os.path.join(src_in_project_builder, project_repo_name)), '-v',
'%s:%s' % (out_dir, '/out')
]
bash_command = 'compile'
command.extend([
'gcr.io/oss-fuzz/' + project_name,
'/bin/bash',
'-c',
])
command.append(bash_command)
if helper.docker_run(command):
logging.error('Building fuzzers failed.')
return False
remove_unaffected_fuzzers(project_name, out_dir,
build_repo_manager.get_git_diff(),
project_builder_repo_path)
return True
def build_fuzzers(project_name,
project_repo_name,
workspace,
pr_ref=None,
commit_sha=None):
"""Builds all of the fuzzers for a specific OSS-Fuzz project.
Args:
project_name: The name of the OSS-Fuzz project being built.
project_repo_name: The name of the projects repo.
workspace: The location in a shared volume to store a git repo and build
artifacts.
pr_ref: The pull request reference to be built.
commit_sha: The commit sha for the project to be built at.
Returns:
True if build succeeded or False on failure.
"""
# Validate inputs.
assert pr_ref or commit_sha
if not os.path.exists(workspace):
logging.error('Invalid workspace: %s.', workspace)
return False
git_workspace = os.path.join(workspace, 'storage')
os.makedirs(git_workspace, exist_ok=True)
out_dir = os.path.join(workspace, 'out')
os.makedirs(out_dir, exist_ok=True)
# Detect repo information.
inferred_url, oss_fuzz_repo_path = build_specified_commit.detect_main_repo(
project_name, repo_name=project_repo_name)
if not inferred_url or not oss_fuzz_repo_path:
logging.error('Could not detect repo from project %s.', project_name)
return False
src_in_docker = os.path.dirname(oss_fuzz_repo_path)
oss_fuzz_repo_name = os.path.basename(oss_fuzz_repo_path)
# Checkout projects repo in the shared volume.
build_repo_manager = repo_manager.RepoManager(inferred_url,
git_workspace,
repo_name=oss_fuzz_repo_name)
try:
if pr_ref:
build_repo_manager.checkout_pr(pr_ref)
else:
build_repo_manager.checkout_commit(commit_sha)
except RuntimeError:
logging.error('Can not check out requested state.')
return False
except ValueError:
logging.error('Invalid commit SHA requested %s.', commit_sha)
return False
# Build Fuzzers using docker run.
command = [
'--cap-add', 'SYS_PTRACE', '-e', 'FUZZING_ENGINE=' + DEFAULT_ENGINE,
'-e', 'SANITIZER=' + DEFAULT_SANITIZER, '-e',
'ARCHITECTURE=' + DEFAULT_ARCHITECTURE
]
container = utils.get_container_name()
if container:
command += ['-e', 'OUT=' + out_dir, '--volumes-from', container]
bash_command = 'rm -rf {0} && cp -r {1} {2} && compile'.format(
os.path.join(src_in_docker, oss_fuzz_repo_name, '*'),
os.path.join(git_workspace, oss_fuzz_repo_name), src_in_docker)
else:
command += [
'-e', 'OUT=' + '/out', '-v',
'%s:%s' % (os.path.join(git_workspace, oss_fuzz_repo_name),
os.path.join(src_in_docker, oss_fuzz_repo_name)), '-v',
'%s:%s' % (out_dir, '/out')
]
bash_command = 'compile'
command.extend([
'gcr.io/oss-fuzz/' + project_name,
'/bin/bash',
'-c',
])
command.append(bash_command)
if helper.docker_run(command):
logging.error('Building fuzzers failed.')
return False
remove_unaffected_fuzzers(project_name, out_dir,
build_repo_manager.get_git_diff(),
oss_fuzz_repo_path)
return True
def build_fuzzers_from_commit(commit,
build_repo_manager,
host_src_path,
build_data,
base_builder_repo=None):
"""Builds a OSS-Fuzz fuzzer at a specific commit SHA.
Args:
commit: The commit SHA to build the fuzzers at.
build_repo_manager: The OSS-Fuzz project's repo manager to be built at.
build_data: A struct containing project build information.
base_builder_repo: A BaseBuilderRepo.
Returns:
0 on successful build or error code on failure.
"""
oss_fuzz_repo_manager = repo_manager.RepoManager(helper.OSS_FUZZ_DIR)
num_retry = 1
def cleanup():
# Re-copy /src for a clean checkout every time.
copy_src_from_docker(build_data.project_name,
os.path.dirname(host_src_path))
projects_dir = os.path.join('projects', build_data.project_name)
dockerfile_path = os.path.join(projects_dir, 'Dockerfile')
for i in range(num_retry + 1):
build_repo_manager.checkout_commit(commit, clean=False)
post_checkout_steps = get_required_post_checkout_steps(dockerfile_path)
for workdir, post_checkout_step in post_checkout_steps:
logging.info('Running post-checkout step `%s` in %s.', post_checkout_step,
workdir)
helper.docker_run([
'-w',
workdir,
'-v',
host_src_path + ':' + '/src',
'gcr.io/oss-fuzz/' + build_data.project_name,
'/bin/bash',
'-c',
post_checkout_step,
])
result = helper.build_fuzzers_impl(project_name=build_data.project_name,
clean=True,
engine=build_data.engine,
sanitizer=build_data.sanitizer,
architecture=build_data.architecture,
env_to_add=None,
source_path=host_src_path,
mount_location='/src')
if result == 0 or i == num_retry:
break
# Retry with an OSS-Fuzz builder container that's closer to the project
# commit date.
commit_date = build_repo_manager.commit_date(commit)
# Find first change in the projects/<PROJECT> directory before the project
# commit date.
oss_fuzz_commit, _, _ = oss_fuzz_repo_manager.git([
'log', '--before=' + commit_date.isoformat(), '-n1', '--format=%H',
projects_dir
],
check_result=True)
oss_fuzz_commit = oss_fuzz_commit.strip()
if not oss_fuzz_commit:
logging.info(
'Could not find first OSS-Fuzz commit prior to upstream commit. '
'Falling back to oldest integration commit.')
# Find the oldest commit.
oss_fuzz_commit, _, _ = oss_fuzz_repo_manager.git(
['log', '--reverse', '--format=%H', projects_dir], check_result=True)
oss_fuzz_commit = oss_fuzz_commit.splitlines()[0].strip()
if not oss_fuzz_commit:
logging.error('Failed to get oldest integration commit.')
break
logging.info('Build failed. Retrying on earlier OSS-Fuzz commit %s.',
oss_fuzz_commit)
# Check out projects/<PROJECT> dir to the commit that was found.
oss_fuzz_repo_manager.git(['checkout', oss_fuzz_commit, projects_dir],
check_result=True)
# Also use the closest base-builder we can find.
if base_builder_repo:
base_builder_digest = base_builder_repo.find_digest(commit_date)
logging.info('Using base-builder with digest %s.', base_builder_digest)
_replace_base_builder_digest(dockerfile_path, base_builder_digest)
# Rebuild image and re-copy src dir since things in /src could have changed.
if not _build_image_with_retries(build_data.project_name):
raise RuntimeError('Failed to rebuild image.')
cleanup()
cleanup()
return result == 0
def build_fuzzers(project_name, project_repo_name, commit_sha, git_workspace,
out_dir):
"""Builds all of the fuzzers for a specific OSS-Fuzz project.
Args:
project_name: The name of the OSS-Fuzz project being built.
project_repo_name: The name of the projects repo.
commit_sha: The commit SHA to be checked out and fuzzed.
git_workspace: The location in the shared volume to store git repos.
out_dir: The location in the shared volume to store output artifacts.
Returns:
True if build succeeded or False on failure.
"""
if not os.path.exists(git_workspace):
logging.error('Invalid git workspace: %s.', format(git_workspace))
return False
if not os.path.exists(out_dir):
logging.error('Invalid out directory %s.', format(out_dir))
return False
inferred_url, oss_fuzz_repo_path = build_specified_commit.detect_main_repo(
project_name, repo_name=project_repo_name)
if not inferred_url or not oss_fuzz_repo_path:
logging.error('Could not detect repo from project %s.', project_name)
return False
src_in_docker = os.path.dirname(oss_fuzz_repo_path)
oss_fuzz_repo_name = os.path.basename(oss_fuzz_repo_path)
# Checkout projects repo in the shared volume.
build_repo_manager = repo_manager.RepoManager(inferred_url,
git_workspace,
repo_name=oss_fuzz_repo_name)
try:
build_repo_manager.checkout_commit(commit_sha)
except repo_manager.RepoManagerError:
logging.error('Specified commit does not exist.')
# NOTE: Remove return statement for testing.
return False
command = [
'--cap-add', 'SYS_PTRACE', '-e', 'FUZZING_ENGINE=libfuzzer', '-e',
'SANITIZER=address', '-e', 'ARCHITECTURE=x86_64'
]
container = utils.get_container_name()
if container:
command += ['-e', 'OUT=' + out_dir, '--volumes-from', container]
bash_command = 'rm -rf {0} && cp -r {1} {2} && compile'.format(
os.path.join(src_in_docker, oss_fuzz_repo_name, '*'),
os.path.join(git_workspace, oss_fuzz_repo_name), src_in_docker)
else:
command += [
'-e', 'OUT=' + '/out', '-v',
'%s:%s' % (os.path.join(git_workspace, oss_fuzz_repo_name),
os.path.join(src_in_docker, oss_fuzz_repo_name)), '-v',
'%s:%s' % (out_dir, '/out')
]
bash_command = 'compile'
command.extend([
'gcr.io/oss-fuzz/' + project_name,
'/bin/bash',
'-c',
])
command.append(bash_command)
if helper.docker_run(command):
logging.error('Building fuzzers failed.')
return False
return True
