def build_measurer(benchmark: str) -> bool: """Do a coverage build for a benchmark.""" try: logger.info('Building measurer for benchmark: %s.', benchmark) buildlib.build_coverage(benchmark) docker_name = benchmark_utils.get_docker_name(benchmark) archive_name = 'coverage-build-%s.tar.gz' % docker_name coverage_binaries_dir = build_utils.get_coverage_binaries_dir() benchmark_coverage_binary_dir = coverage_binaries_dir / benchmark os.mkdir(benchmark_coverage_binary_dir) cloud_bucket_archive_path = exp_path.gcs(coverage_binaries_dir / archive_name) gsutil.cp(cloud_bucket_archive_path, str(benchmark_coverage_binary_dir), parallel=False, write_to_stdout=False) archive_path = benchmark_coverage_binary_dir / archive_name tar = tarfile.open(archive_path, 'r:gz') tar.extractall(benchmark_coverage_binary_dir) os.remove(archive_path) logs.info('Done building measurer for benchmark: %s.', benchmark) return True except Exception: # pylint: disable=broad-except logger.error('Failed to build measurer for %s.', benchmark) return False
def get_coverage_binary(benchmark: str) -> str: """Get the coverage binary for benchmark.""" coverage_binaries_dir = build_utils.get_coverage_binaries_dir() fuzz_target = benchmark_utils.get_fuzz_target(benchmark) return fuzzer_utils.get_fuzz_target_binary(coverage_binaries_dir / benchmark, fuzz_target_name=fuzz_target)
def coverage_steps(benchmark): """Returns GCB run steps for coverage builds.""" coverage_binaries_dir = exp_path.filestore( build_utils.get_coverage_binaries_dir()) steps = [{ 'name': DOCKER_IMAGE, 'args': [ 'run', '-v', '/workspace/out:/host-out', # TODO(metzman): Get rid of this and use one source of truth # for tags. posixpath.join(get_docker_registry(), 'builders', 'coverage', benchmark) + ':' + experiment_utils.get_experiment_name(), '/bin/bash', '-c', 'cd /out; tar -czvf /host-out/coverage-build-' + benchmark + '.tar.gz * /src /work' ] }] step = {'name': 'gcr.io/cloud-builders/gsutil'} step['args'] = [ '-m', 'cp', '/workspace/out/coverage-build-' + benchmark + '.tar.gz', coverage_binaries_dir + '/' ] steps.append(step) return steps
def __init__(self, experiment, fuzzer, benchmark): self.fuzzer = fuzzer self.benchmark = benchmark self.experiment = experiment self.trial_ids = get_trial_ids(experiment, fuzzer, benchmark) coverage_info_dir = get_coverage_info_dir() self.report_dir = os.path.join(coverage_info_dir, 'reports', benchmark, fuzzer) self.data_dir = os.path.join(coverage_info_dir, 'data', benchmark, fuzzer) benchmark_fuzzer_dir = exp_utils.get_benchmark_fuzzer_dir( benchmark, fuzzer) work_dir = exp_utils.get_work_dir() benchmark_fuzzer_measurement_dir = os.path.join( work_dir, 'measurement-folders', benchmark_fuzzer_dir) self.merged_profdata_file = os.path.join( benchmark_fuzzer_measurement_dir, 'merged.profdata') self.merged_summary_json_file = os.path.join( benchmark_fuzzer_measurement_dir, 'merged.json') coverage_binaries_dir = build_utils.get_coverage_binaries_dir() self.source_files_dir = os.path.join(coverage_binaries_dir, benchmark) self.binary_file = get_coverage_binary(benchmark)
def create_measurer(experiment): """Fixture that provides a function for creating a SnapshotMeasurer.""" os.mkdir(build_utils.get_coverage_binaries_dir()) def _create_measurer(fuzzer, benchmark, trial_num): return measurer.SnapshotMeasurer(fuzzer, benchmark, trial_num, SNAPSHOT_LOGGER) yield _create_measurer
def build_all_measurers(benchmarks: List[str]) -> List[str]: """Build measurers for benchmarks.""" logger.info('Building measurers.') filesystem.recreate_directory(build_utils.get_coverage_binaries_dir()) benchmarks = [(benchmark, ) for benchmark in benchmarks] results = retry_build_loop(build_measurer, benchmarks) logger.info('Done building measurers.') # Return list of benchmarks (like the list we were passed as an argument) # instead of returning a list of tuples each containing a benchmark. return [result[0] for result in results]
def set_up_coverage_binaries(pool, experiment): """Set up coverage binaries for all benchmarks in |experiment|.""" benchmarks = [ trial.benchmark for trial in db_utils.query(models.Trial).distinct( models.Trial.benchmark).filter( models.Trial.experiment == experiment) ] coverage_binaries_dir = build_utils.get_coverage_binaries_dir() if not os.path.exists(coverage_binaries_dir): os.makedirs(coverage_binaries_dir) pool.map(set_up_coverage_binary, benchmarks)
def build_coverage(benchmark): """Build coverage image for benchmark on GCB.""" coverage_binaries_dir = exp_path.filestore( build_utils.get_coverage_binaries_dir()) substitutions = { '_GCS_COVERAGE_BINARIES_DIR': coverage_binaries_dir, '_BENCHMARK': benchmark, } config_file = get_build_config_file('coverage.yaml') config_name = 'benchmark-{benchmark}-coverage'.format(benchmark=benchmark) _build(config_file, config_name, substitutions)
def build_all_measurers(benchmarks: List[str]) -> List[str]: """Build measurers for each benchmark in |benchmarks| in parallel Returns a list of benchmarks built successfully.""" logger.info('Building measurers.') filesystem.recreate_directory(build_utils.get_coverage_binaries_dir()) build_measurer_args = [(benchmark, ) for benchmark in benchmarks] successful_calls = retry_build_loop(build_measurer, build_measurer_args) logger.info('Done building measurers.') # Return list of benchmarks (like the list we were passed as an argument) # instead of returning a list of tuples each containing a benchmark. return [successful_call[0] for successful_call in successful_calls]
def _build_benchmark_coverage(benchmark: str) -> Tuple[int, str]: """Build a coverage build of |benchmark| on GCB.""" coverage_binaries_dir = exp_path.gcs( build_utils.get_coverage_binaries_dir()) substitutions = { '_GCS_COVERAGE_BINARIES_DIR': coverage_binaries_dir, '_BENCHMARK': benchmark, } config_file = get_build_config_file('coverage.yaml') config_name = 'benchmark-{benchmark}-coverage'.format(benchmark=benchmark) return _build(config_file, config_name, substitutions)
def test_measure_snapshot_coverage( # pylint: disable=too-many-locals self, mocked_is_cycle_unchanged, db, experiment, tmp_path): """Integration test for measure_snapshot_coverage.""" # WORK is set by experiment to a directory that only makes sense in a # fakefs. A directory containing necessary llvm tools is also added to # PATH. llvm_tools_path = get_test_data_path('llvm_tools') os.environ["PATH"] += os.pathsep + llvm_tools_path os.environ['WORK'] = str(tmp_path) mocked_is_cycle_unchanged.return_value = False # Set up the coverage binary. benchmark = 'freetype2-2017' coverage_binary_src = get_test_data_path( 'test_measure_snapshot_coverage', benchmark + '-coverage') benchmark_cov_binary_dir = os.path.join( build_utils.get_coverage_binaries_dir(), benchmark) os.makedirs(benchmark_cov_binary_dir) coverage_binary_dst_dir = os.path.join(benchmark_cov_binary_dir, 'ftfuzzer') shutil.copy(coverage_binary_src, coverage_binary_dst_dir) # Set up entities in database so that the snapshot can be created. experiment = models.Experiment(name=os.environ['EXPERIMENT']) db_utils.add_all([experiment]) trial = models.Trial(fuzzer=FUZZER, benchmark=benchmark, experiment=os.environ['EXPERIMENT']) db_utils.add_all([trial]) snapshot_measurer = measurer.SnapshotMeasurer(trial.fuzzer, trial.benchmark, trial.id, SNAPSHOT_LOGGER) # Set up the snapshot archive. cycle = 1 archive = get_test_data_path('test_measure_snapshot_coverage', 'corpus-archive-%04d.tar.gz' % cycle) corpus_dir = os.path.join(snapshot_measurer.trial_dir, 'corpus') os.makedirs(corpus_dir) shutil.copy(archive, corpus_dir) with mock.patch('common.filestore_utils.cp') as mocked_cp: mocked_cp.return_value = new_process.ProcessResult(0, '', False) # TODO(metzman): Create a system for using actual buckets in # integration tests. snapshot = measurer.measure_snapshot_coverage( snapshot_measurer.fuzzer, snapshot_measurer.benchmark, snapshot_measurer.trial_num, cycle) assert snapshot assert snapshot.time == cycle * experiment_utils.get_snapshot_seconds() assert snapshot.edges_covered == 13178
def _build_oss_fuzz_project_coverage(benchmark: str) -> Tuple[int, str]: """Build a coverage build of OSS-Fuzz-based benchmark |benchmark| on GCB.""" coverage_binaries_dir = exp_path.filestore( build_utils.get_coverage_binaries_dir()) substitutions = { '_GCS_COVERAGE_BINARIES_DIR': coverage_binaries_dir, '_BENCHMARK': benchmark, } config_file = get_build_config_file('oss-fuzz-coverage.yaml') config_name = 'oss-fuzz-{benchmark}-coverage'.format(benchmark=benchmark) return _build(config_file, config_name, substitutions)
def set_up_coverage_binaries(pool, experiment): """Set up coverage binaries for all benchmarks in |experiment|.""" # Use set comprehension to select distinct benchmarks. benchmarks = [ benchmark_tuple[0] for benchmark_tuple in db_utils.query(models.Trial.benchmark).distinct( ).filter(models.Trial.experiment == experiment) ] coverage_binaries_dir = build_utils.get_coverage_binaries_dir() filesystem.create_directory(coverage_binaries_dir) pool.map(set_up_coverage_binary, benchmarks)
def set_up_coverage_binary(benchmark): """Set up coverage binaries for |benchmark|.""" initialize_logs() coverage_binaries_dir = build_utils.get_coverage_binaries_dir() benchmark_coverage_binary_dir = coverage_binaries_dir / benchmark filesystem.create_directory(benchmark_coverage_binary_dir) archive_name = 'coverage-build-%s.tar.gz' % benchmark archive_filestore_path = exp_path.filestore(coverage_binaries_dir / archive_name) filestore_utils.cp(archive_filestore_path, str(benchmark_coverage_binary_dir)) archive_path = benchmark_coverage_binary_dir / archive_name tar = tarfile.open(archive_path, 'r:gz') tar.extractall(benchmark_coverage_binary_dir) os.remove(archive_path)
def _build_oss_fuzz_project_coverage(benchmark: str) -> Tuple[int, str]: """Build a coverage build of OSS-Fuzz-based benchmark |benchmark| on GCB.""" project = benchmark_utils.get_project(benchmark) oss_fuzz_builder_hash = benchmark_utils.get_oss_fuzz_builder_hash(benchmark) coverage_binaries_dir = exp_path.gcs( build_utils.get_coverage_binaries_dir()) substitutions = { '_GCS_COVERAGE_BINARIES_DIR': coverage_binaries_dir, '_OSS_FUZZ_PROJECT': project, '_OSS_FUZZ_BUILDER_HASH': oss_fuzz_builder_hash, } config_file = get_build_config_file('oss-fuzz-coverage.yaml') config_name = 'oss-fuzz-{project}-coverage-hash-{hash}'.format( project=project, hash=oss_fuzz_builder_hash) return _build(config_file, config_name, substitutions)
def test_measure_snapshot_coverage( # pylint: disable=too-many-locals self, mocked_is_cycle_unchanged, create_measurer, db, experiment): """Integration test for measure_snapshot_coverage.""" mocked_is_cycle_unchanged.return_value = False # Set up the coverage binary. benchmark = 'freetype2-2017' coverage_binary_src = get_test_data_path( 'test_measure_snapshot_coverage', benchmark + '-coverage') benchmark_cov_binary_dir = os.path.join( build_utils.get_coverage_binaries_dir(), benchmark) os.makedirs(benchmark_cov_binary_dir) coverage_binary_dst_dir = os.path.join(benchmark_cov_binary_dir, 'fuzz-target') shutil.copy(coverage_binary_src, coverage_binary_dst_dir) # Set up entities in database so that the snapshot can be created. experiment = models.Experiment(name=os.environ['EXPERIMENT']) db_utils.add_all([experiment]) trial = models.Trial(fuzzer=FUZZER, benchmark=benchmark, experiment=os.environ['EXPERIMENT']) db_utils.add_all([trial]) snapshot_measurer = create_measurer(trial.fuzzer, trial.benchmark, trial.id) # Set up the snapshot archive. cycle = 1 archive = get_test_data_path('test_measure_snapshot_coverage', 'corpus-archive-%04d.tar.gz' % cycle) corpus_dir = os.path.join(snapshot_measurer.trial_dir, 'corpus') os.makedirs(corpus_dir) shutil.copy(archive, corpus_dir) with mock.patch('common.gsutil.cp') as mocked_cp: mocked_cp.return_value = new_process.ProcessResult(0, '', False) # TODO(metzman): Create a system for using actual buckets in # integration tests. snapshot = measurer.measure_snapshot_coverage( snapshot_measurer.fuzzer, snapshot_measurer.benchmark, snapshot_measurer.trial_num, cycle) assert snapshot assert snapshot.time == cycle * experiment_utils.get_snapshot_seconds() assert snapshot.edges_covered == 3798
def set_up_coverage_binary(benchmark): """Set up coverage binaries for |benchmark|.""" initialize_logs() coverage_binaries_dir = build_utils.get_coverage_binaries_dir() benchmark_coverage_binary_dir = coverage_binaries_dir / benchmark if not os.path.exists(benchmark_coverage_binary_dir): os.mkdir(benchmark_coverage_binary_dir) archive_name = 'coverage-build-%s.tar.gz' % benchmark cloud_bucket_archive_path = exp_path.gcs(coverage_binaries_dir / archive_name) gsutil.cp(cloud_bucket_archive_path, str(benchmark_coverage_binary_dir), write_to_stdout=False) archive_path = benchmark_coverage_binary_dir / archive_name tar = tarfile.open(archive_path, 'r:gz') tar.extractall(benchmark_coverage_binary_dir) os.remove(archive_path)
def copy_coverage_binaries(benchmark): """Copy coverage binaries in a local experiment.""" shared_coverage_binaries_dir = get_shared_coverage_binaries_dir() mount_arg = '{0}:{0}'.format(shared_coverage_binaries_dir) builder_image_url = benchmark_utils.get_builder_image_url( benchmark, 'coverage', environment.get('CLOUD_PROJECT')) coverage_build_archive = 'coverage-build-{}.tar.gz'.format(benchmark) coverage_build_archive_shared_dir_path = os.path.join( shared_coverage_binaries_dir, coverage_build_archive) command = 'cd /out; tar -czvf {} *'.format( coverage_build_archive_shared_dir_path) new_process.execute([ 'docker', 'run', '-v', mount_arg, builder_image_url, '/bin/bash', '-c', command ]) coverage_binaries_dir = build_utils.get_coverage_binaries_dir() coverage_build_archive_gcs_path = posixpath.join( exp_path.gcs(coverage_binaries_dir), coverage_build_archive) return gsutil.cp(coverage_build_archive_shared_dir_path, coverage_build_archive_gcs_path)