def get_build_steps( # pylint: disable=too-many-locals, too-many-arguments
project_name, project_yaml, dockerfile_lines, image_project,
base_images_project, config):
"""Returns build steps for project."""
project = build_project.Project(project_name, project_yaml,
dockerfile_lines, image_project)
if project.disabled:
logging.info('Project "%s" is disabled.', project.name)
return []
if project.fuzzing_language not in LANGUAGES_WITH_COVERAGE_SUPPORT:
logging.info(
'Project "%s" is written in "%s", coverage is not supported yet.',
project.name, project.fuzzing_language)
return []
report_date = build_project.get_datetime_now().strftime('%Y%m%d')
bucket = CoverageBucket(project.name, report_date, PLATFORM,
config.testing)
build_steps = build_lib.project_image_steps(
project.name,
project.image,
project.fuzzing_language,
branch=config.branch,
test_image_suffix=config.test_image_suffix)
build = build_project.Build(FUZZING_ENGINE, 'coverage', ARCHITECTURE)
env = build_project.get_env(project.fuzzing_language, build)
build_steps.append(
build_project.get_compile_step(project, build, env, config.parallel))
download_corpora_steps = build_lib.download_corpora_steps(
project.name, testing=config.testing)
if not download_corpora_steps:
logging.info('Skipping code coverage build for %s.', project.name)
return []
build_steps.extend(download_corpora_steps)
failure_msg = ('*' * 80 + '\nCode coverage report generation failed.\n'
'To reproduce, run:\n'
f'python infra/helper.py build_image {project.name}\n'
'python infra/helper.py build_fuzzers --sanitizer coverage '
f'{project.name}\n'
f'python infra/helper.py coverage {project.name}\n' +
'*' * 80)
# Unpack the corpus and run coverage script.
coverage_env = env + [
'HTTP_PORT=',
f'COVERAGE_EXTRA_ARGS={project.coverage_extra_args.strip()}',
]
if 'dataflow' in project.fuzzing_engines:
coverage_env.append('FULL_SUMMARY_PER_TARGET=1')
build_steps.append({
'name':
build_project.get_runner_image_name(base_images_project,
config.test_image_suffix),
'env':
coverage_env,
'args': [
'bash', '-c',
('for f in /corpus/*.zip; do unzip -q $f -d ${f%%.*} || ('
'echo "Failed to unpack the corpus for $(basename ${f%%.*}). '
'This usually means that corpus backup for a particular fuzz '
'target does not exist. If a fuzz target was added in the last '
'24 hours, please wait one more day. Otherwise, something is '
'wrong with the fuzz target or the infrastructure, and corpus '
'pruning task does not finish successfully." && exit 1'
'); done && coverage || (echo "' + failure_msg + '" && false)')
],
'volumes': [{
'name': 'corpus',
'path': '/corpus'
}],
})
# Upload the report.
upload_report_url = bucket.get_upload_url('reports')
upload_report_by_target_url = bucket.get_upload_url('reports-by-target')
# Delete the existing report as gsutil cannot overwrite it in a useful way due
# to the lack of `-T` option (it creates a subdir in the destination dir).
build_steps.append(build_lib.gsutil_rm_rf_step(upload_report_url))
build_steps.append({
'name':
'gcr.io/cloud-builders/gsutil',
'args': [
'-m',
'cp',
'-r',
os.path.join(build.out, 'report'),
upload_report_url,
],
})
if project.fuzzing_language in LANGUAGES_WITH_INTROSPECTOR_SUPPORT:
build_steps.append(
build_lib.gsutil_rm_rf_step(upload_report_by_target_url))
build_steps.append({
'name':
'gcr.io/cloud-builders/gsutil',
'args': [
'-m',
'cp',
'-r',
os.path.join(build.out, 'report_target'),
upload_report_by_target_url,
],
})
# Upload the fuzzer stats. Delete the old ones just in case.
upload_fuzzer_stats_url = bucket.get_upload_url('fuzzer_stats')
build_steps.append(build_lib.gsutil_rm_rf_step(upload_fuzzer_stats_url))
build_steps.append({
'name':
'gcr.io/cloud-builders/gsutil',
'args': [
'-m',
'cp',
'-r',
os.path.join(build.out, 'fuzzer_stats'),
upload_fuzzer_stats_url,
],
})
if project.fuzzing_language in LANGUAGES_WITH_INTROSPECTOR_SUPPORT:
# Upload the text coverage reports. Delete the old ones just in case.
upload_textcov_reports_url = bucket.get_upload_url('textcov_reports')
build_steps.append(
build_lib.gsutil_rm_rf_step(upload_textcov_reports_url))
build_steps.append({
'name':
'gcr.io/cloud-builders/gsutil',
'args': [
'-m',
'cp',
'-r',
os.path.join(build.out, 'textcov_reports'),
upload_textcov_reports_url,
],
})
# Upload the fuzzer logs. Delete the old ones just in case
upload_fuzzer_logs_url = bucket.get_upload_url('logs')
build_steps.append(build_lib.gsutil_rm_rf_step(upload_fuzzer_logs_url))
build_steps.append({
'name':
'gcr.io/cloud-builders/gsutil',
'args': [
'-m',
'cp',
'-r',
os.path.join(build.out, 'logs'),
upload_fuzzer_logs_url,
],
})
# Upload srcmap.
srcmap_upload_url = bucket.get_upload_url('srcmap')
srcmap_upload_url = srcmap_upload_url.rstrip('/') + '.json'
build_steps.append({
'name':
'gcr.io/cloud-builders/gsutil',
'args': [
'cp',
'/workspace/srcmap.json',
srcmap_upload_url,
],
})
# Update the latest report information file for ClusterFuzz.
latest_report_info_url = build_lib.get_signed_url(
bucket.latest_report_info_url,
content_type=LATEST_REPORT_INFO_CONTENT_TYPE)
latest_report_info_body = json.dumps({
'fuzzer_stats_dir':
upload_fuzzer_stats_url,
'html_report_url':
posixpath.join(bucket.html_report_url, 'index.html'),
'report_date':
report_date,
'report_summary_path':
os.path.join(upload_report_url, PLATFORM, 'summary.json'),
})
build_steps.append(
build_lib.http_upload_step(latest_report_info_body,
latest_report_info_url,
LATEST_REPORT_INFO_CONTENT_TYPE))
return build_steps
def get_fuzz_introspector_steps( # pylint: disable=too-many-locals, too-many-arguments, unused-argument
project_name, project_yaml, dockerfile_lines, image_project,
base_images_project, config):
"""Returns build steps of fuzz introspector for project"""
project = build_project.Project(project_name, project_yaml,
dockerfile_lines, image_project)
if project.disabled:
logging.info('Project "%s" is disabled.', project.name)
return []
if project.fuzzing_language not in LANGUAGES_WITH_INTROSPECTOR_SUPPORT:
logging.info(('Project "%s" is written in "%s", '
'Fuzz Introspector is not supported yet.'), project.name,
project.fuzzing_language)
return []
build_steps = []
build = build_project.Build(FUZZING_ENGINE, 'introspector', ARCHITECTURE)
env = build_project.get_env(project.fuzzing_language, build)
report_date = build_project.get_datetime_now().strftime('%Y%m%d')
bucket = IntrospectorBucket(project.name, report_date, PLATFORM,
config.testing)
# TODO (navidem): find the latest coverage report.
coverage_report_latest = report_date
build_steps.append({
'args':
['clone', 'https://github.com/google/oss-fuzz.git', '--depth', '1'],
'name':
'gcr.io/cloud-builders/git',
})
bucket_name = 'oss-fuzz-coverage'
if config.testing:
bucket_name += '-testing'
coverage_url = (f'{build_lib.GCS_URL_BASENAME}{bucket_name}/{project.name}'
f'/reports/{coverage_report_latest}/linux')
download_coverage_steps = build_lib.download_coverage_data_steps(
project.name, coverage_report_latest, bucket_name, build.out,
config.testing)
if not download_coverage_steps:
logging.warning(
'Skipping introspector build for %s. No coverage data found.',
project.name)
return []
build_steps.extend(download_coverage_steps)
build_steps.append({
'name':
'gcr.io/cloud-builders/docker',
'args': ['pull', 'gcr.io/oss-fuzz-base/base-builder:introspector'],
})
build_steps.append({
'name':
'gcr.io/cloud-builders/docker',
'args': [
'tag', 'gcr.io/oss-fuzz-base/base-builder:introspector',
'gcr.io/oss-fuzz-base/base-builder:latest'
],
})
build_steps.append({
'name':
'gcr.io/cloud-builders/docker',
'args': [
'build',
'-t',
f'gcr.io/oss-fuzz/{project.name}',
'.',
],
'dir':
os.path.join('oss-fuzz', 'projects', project.name),
})
env.append(f'GIT_REPO={project.main_repo}')
env.append(f'COVERAGE_URL={coverage_url}')
build_steps.append(
build_project.get_compile_step(project, build, env, config.parallel))
# Upload the report.
upload_report_url = bucket.get_upload_url('inspector-report')
# Delete the existing report as gsutil cannot overwrite it in a useful way due
# to the lack of `-T` option (it creates a subdir in the destination dir).
build_steps.append(build_lib.gsutil_rm_rf_step(upload_report_url))
build_steps.append({
'name':
'gcr.io/cloud-builders/gsutil',
'args': [
'-m',
'cp',
'-r',
os.path.join(build.out, 'inspector'),
upload_report_url,
],
})
return build_steps