コード例 #1
0
def get_build_steps(  # pylint: disable=too-many-locals, too-many-arguments
        project_name, project_yaml, dockerfile_lines, image_project,
        base_images_project, config):
    """Returns build steps for project."""
    project = build_project.Project(project_name, project_yaml,
                                    dockerfile_lines, image_project)
    if project.disabled:
        logging.info('Project "%s" is disabled.', project.name)
        return []

    if project.fuzzing_language not in LANGUAGES_WITH_COVERAGE_SUPPORT:
        logging.info(
            'Project "%s" is written in "%s", coverage is not supported yet.',
            project.name, project.fuzzing_language)
        return []

    report_date = build_project.get_datetime_now().strftime('%Y%m%d')
    bucket = CoverageBucket(project.name, report_date, PLATFORM,
                            config.testing)

    build_steps = build_lib.project_image_steps(
        project.name,
        project.image,
        project.fuzzing_language,
        branch=config.branch,
        test_image_suffix=config.test_image_suffix)

    build = build_project.Build(FUZZING_ENGINE, 'coverage', ARCHITECTURE)
    env = build_project.get_env(project.fuzzing_language, build)
    build_steps.append(
        build_project.get_compile_step(project, build, env, config.parallel))
    download_corpora_steps = build_lib.download_corpora_steps(
        project.name, testing=config.testing)
    if not download_corpora_steps:
        logging.info('Skipping code coverage build for %s.', project.name)
        return []

    build_steps.extend(download_corpora_steps)

    failure_msg = ('*' * 80 + '\nCode coverage report generation failed.\n'
                   'To reproduce, run:\n'
                   f'python infra/helper.py build_image {project.name}\n'
                   'python infra/helper.py build_fuzzers --sanitizer coverage '
                   f'{project.name}\n'
                   f'python infra/helper.py coverage {project.name}\n' +
                   '*' * 80)

    # Unpack the corpus and run coverage script.
    coverage_env = env + [
        'HTTP_PORT=',
        f'COVERAGE_EXTRA_ARGS={project.coverage_extra_args.strip()}',
    ]
    if 'dataflow' in project.fuzzing_engines:
        coverage_env.append('FULL_SUMMARY_PER_TARGET=1')

    build_steps.append({
        'name':
        build_project.get_runner_image_name(base_images_project,
                                            config.test_image_suffix),
        'env':
        coverage_env,
        'args': [
            'bash', '-c',
            ('for f in /corpus/*.zip; do unzip -q $f -d ${f%%.*} || ('
             'echo "Failed to unpack the corpus for $(basename ${f%%.*}). '
             'This usually means that corpus backup for a particular fuzz '
             'target does not exist. If a fuzz target was added in the last '
             '24 hours, please wait one more day. Otherwise, something is '
             'wrong with the fuzz target or the infrastructure, and corpus '
             'pruning task does not finish successfully." && exit 1'
             '); done && coverage || (echo "' + failure_msg + '" && false)')
        ],
        'volumes': [{
            'name': 'corpus',
            'path': '/corpus'
        }],
    })

    # Upload the report.
    upload_report_url = bucket.get_upload_url('reports')
    upload_report_by_target_url = bucket.get_upload_url('reports-by-target')

    # Delete the existing report as gsutil cannot overwrite it in a useful way due
    # to the lack of `-T` option (it creates a subdir in the destination dir).
    build_steps.append(build_lib.gsutil_rm_rf_step(upload_report_url))
    build_steps.append({
        'name':
        'gcr.io/cloud-builders/gsutil',
        'args': [
            '-m',
            'cp',
            '-r',
            os.path.join(build.out, 'report'),
            upload_report_url,
        ],
    })

    if project.fuzzing_language in LANGUAGES_WITH_INTROSPECTOR_SUPPORT:
        build_steps.append(
            build_lib.gsutil_rm_rf_step(upload_report_by_target_url))
        build_steps.append({
            'name':
            'gcr.io/cloud-builders/gsutil',
            'args': [
                '-m',
                'cp',
                '-r',
                os.path.join(build.out, 'report_target'),
                upload_report_by_target_url,
            ],
        })

    # Upload the fuzzer stats. Delete the old ones just in case.
    upload_fuzzer_stats_url = bucket.get_upload_url('fuzzer_stats')

    build_steps.append(build_lib.gsutil_rm_rf_step(upload_fuzzer_stats_url))
    build_steps.append({
        'name':
        'gcr.io/cloud-builders/gsutil',
        'args': [
            '-m',
            'cp',
            '-r',
            os.path.join(build.out, 'fuzzer_stats'),
            upload_fuzzer_stats_url,
        ],
    })

    if project.fuzzing_language in LANGUAGES_WITH_INTROSPECTOR_SUPPORT:
        # Upload the text coverage reports. Delete the old ones just in case.
        upload_textcov_reports_url = bucket.get_upload_url('textcov_reports')

        build_steps.append(
            build_lib.gsutil_rm_rf_step(upload_textcov_reports_url))
        build_steps.append({
            'name':
            'gcr.io/cloud-builders/gsutil',
            'args': [
                '-m',
                'cp',
                '-r',
                os.path.join(build.out, 'textcov_reports'),
                upload_textcov_reports_url,
            ],
        })

    # Upload the fuzzer logs. Delete the old ones just in case
    upload_fuzzer_logs_url = bucket.get_upload_url('logs')
    build_steps.append(build_lib.gsutil_rm_rf_step(upload_fuzzer_logs_url))
    build_steps.append({
        'name':
        'gcr.io/cloud-builders/gsutil',
        'args': [
            '-m',
            'cp',
            '-r',
            os.path.join(build.out, 'logs'),
            upload_fuzzer_logs_url,
        ],
    })

    # Upload srcmap.
    srcmap_upload_url = bucket.get_upload_url('srcmap')
    srcmap_upload_url = srcmap_upload_url.rstrip('/') + '.json'
    build_steps.append({
        'name':
        'gcr.io/cloud-builders/gsutil',
        'args': [
            'cp',
            '/workspace/srcmap.json',
            srcmap_upload_url,
        ],
    })

    # Update the latest report information file for ClusterFuzz.
    latest_report_info_url = build_lib.get_signed_url(
        bucket.latest_report_info_url,
        content_type=LATEST_REPORT_INFO_CONTENT_TYPE)
    latest_report_info_body = json.dumps({
        'fuzzer_stats_dir':
        upload_fuzzer_stats_url,
        'html_report_url':
        posixpath.join(bucket.html_report_url, 'index.html'),
        'report_date':
        report_date,
        'report_summary_path':
        os.path.join(upload_report_url, PLATFORM, 'summary.json'),
    })

    build_steps.append(
        build_lib.http_upload_step(latest_report_info_body,
                                   latest_report_info_url,
                                   LATEST_REPORT_INFO_CONTENT_TYPE))

    return build_steps
コード例 #2
0
def get_fuzz_introspector_steps(  # pylint: disable=too-many-locals, too-many-arguments, unused-argument
        project_name, project_yaml, dockerfile_lines, image_project,
        base_images_project, config):
    """Returns build steps of fuzz introspector for project"""
    project = build_project.Project(project_name, project_yaml,
                                    dockerfile_lines, image_project)
    if project.disabled:
        logging.info('Project "%s" is disabled.', project.name)
        return []

    if project.fuzzing_language not in LANGUAGES_WITH_INTROSPECTOR_SUPPORT:
        logging.info(('Project "%s" is written in "%s", '
                      'Fuzz Introspector is not supported yet.'), project.name,
                     project.fuzzing_language)
        return []

    build_steps = []
    build = build_project.Build(FUZZING_ENGINE, 'introspector', ARCHITECTURE)
    env = build_project.get_env(project.fuzzing_language, build)

    report_date = build_project.get_datetime_now().strftime('%Y%m%d')
    bucket = IntrospectorBucket(project.name, report_date, PLATFORM,
                                config.testing)

    # TODO (navidem): find the latest coverage report.
    coverage_report_latest = report_date
    build_steps.append({
        'args':
        ['clone', 'https://github.com/google/oss-fuzz.git', '--depth', '1'],
        'name':
        'gcr.io/cloud-builders/git',
    })

    bucket_name = 'oss-fuzz-coverage'
    if config.testing:
        bucket_name += '-testing'

    coverage_url = (f'{build_lib.GCS_URL_BASENAME}{bucket_name}/{project.name}'
                    f'/reports/{coverage_report_latest}/linux')

    download_coverage_steps = build_lib.download_coverage_data_steps(
        project.name, coverage_report_latest, bucket_name, build.out,
        config.testing)
    if not download_coverage_steps:
        logging.warning(
            'Skipping introspector build for %s. No coverage data found.',
            project.name)
        return []
    build_steps.extend(download_coverage_steps)

    build_steps.append({
        'name':
        'gcr.io/cloud-builders/docker',
        'args': ['pull', 'gcr.io/oss-fuzz-base/base-builder:introspector'],
    })
    build_steps.append({
        'name':
        'gcr.io/cloud-builders/docker',
        'args': [
            'tag', 'gcr.io/oss-fuzz-base/base-builder:introspector',
            'gcr.io/oss-fuzz-base/base-builder:latest'
        ],
    })

    build_steps.append({
        'name':
        'gcr.io/cloud-builders/docker',
        'args': [
            'build',
            '-t',
            f'gcr.io/oss-fuzz/{project.name}',
            '.',
        ],
        'dir':
        os.path.join('oss-fuzz', 'projects', project.name),
    })

    env.append(f'GIT_REPO={project.main_repo}')
    env.append(f'COVERAGE_URL={coverage_url}')

    build_steps.append(
        build_project.get_compile_step(project, build, env, config.parallel))

    # Upload the report.
    upload_report_url = bucket.get_upload_url('inspector-report')

    # Delete the existing report as gsutil cannot overwrite it in a useful way due
    # to the lack of `-T` option (it creates a subdir in the destination dir).
    build_steps.append(build_lib.gsutil_rm_rf_step(upload_report_url))
    build_steps.append({
        'name':
        'gcr.io/cloud-builders/gsutil',
        'args': [
            '-m',
            'cp',
            '-r',
            os.path.join(build.out, 'inspector'),
            upload_report_url,
        ],
    })

    return build_steps