def build_measurer(benchmark: str) -> bool:
"""Do a coverage build for a benchmark."""
try:
logger.info('Building measurer for benchmark: %s.', benchmark)
buildlib.build_coverage(benchmark)
docker_name = benchmark_utils.get_docker_name(benchmark)
archive_name = 'coverage-build-%s.tar.gz' % docker_name
coverage_binaries_dir = build_utils.get_coverage_binaries_dir()
benchmark_coverage_binary_dir = coverage_binaries_dir / benchmark
os.mkdir(benchmark_coverage_binary_dir)
cloud_bucket_archive_path = exp_path.gcs(coverage_binaries_dir /
archive_name)
gsutil.cp(cloud_bucket_archive_path,
str(benchmark_coverage_binary_dir),
parallel=False,
write_to_stdout=False)
archive_path = benchmark_coverage_binary_dir / archive_name
tar = tarfile.open(archive_path, 'r:gz')
tar.extractall(benchmark_coverage_binary_dir)
os.remove(archive_path)
logs.info('Done building measurer for benchmark: %s.', benchmark)
return True
except Exception: # pylint: disable=broad-except
logger.error('Failed to build measurer for %s.', benchmark)
return False
def get_coverage_binary(benchmark: str) -> str:
"""Get the coverage binary for benchmark."""
coverage_binaries_dir = build_utils.get_coverage_binaries_dir()
fuzz_target = benchmark_utils.get_fuzz_target(benchmark)
return fuzzer_utils.get_fuzz_target_binary(coverage_binaries_dir /
benchmark,
fuzz_target_name=fuzz_target)
def coverage_steps(benchmark):
"""Returns GCB run steps for coverage builds."""
coverage_binaries_dir = exp_path.filestore(
build_utils.get_coverage_binaries_dir())
steps = [{
'name':
DOCKER_IMAGE,
'args': [
'run',
'-v',
'/workspace/out:/host-out',
# TODO(metzman): Get rid of this and use one source of truth
# for tags.
posixpath.join(get_docker_registry(), 'builders', 'coverage',
benchmark) + ':' +
experiment_utils.get_experiment_name(),
'/bin/bash',
'-c',
'cd /out; tar -czvf /host-out/coverage-build-' + benchmark +
'.tar.gz * /src /work'
]
}]
step = {'name': 'gcr.io/cloud-builders/gsutil'}
step['args'] = [
'-m', 'cp', '/workspace/out/coverage-build-' + benchmark + '.tar.gz',
coverage_binaries_dir + '/'
]
steps.append(step)
return steps
def __init__(self, experiment, fuzzer, benchmark):
self.fuzzer = fuzzer
self.benchmark = benchmark
self.experiment = experiment
self.trial_ids = get_trial_ids(experiment, fuzzer, benchmark)
coverage_info_dir = get_coverage_info_dir()
self.report_dir = os.path.join(coverage_info_dir, 'reports', benchmark,
fuzzer)
self.data_dir = os.path.join(coverage_info_dir, 'data', benchmark,
fuzzer)
benchmark_fuzzer_dir = exp_utils.get_benchmark_fuzzer_dir(
benchmark, fuzzer)
work_dir = exp_utils.get_work_dir()
benchmark_fuzzer_measurement_dir = os.path.join(
work_dir, 'measurement-folders', benchmark_fuzzer_dir)
self.merged_profdata_file = os.path.join(
benchmark_fuzzer_measurement_dir, 'merged.profdata')
self.merged_summary_json_file = os.path.join(
benchmark_fuzzer_measurement_dir, 'merged.json')
coverage_binaries_dir = build_utils.get_coverage_binaries_dir()
self.source_files_dir = os.path.join(coverage_binaries_dir, benchmark)
self.binary_file = get_coverage_binary(benchmark)
def create_measurer(experiment):
"""Fixture that provides a function for creating a SnapshotMeasurer."""
os.mkdir(build_utils.get_coverage_binaries_dir())
def _create_measurer(fuzzer, benchmark, trial_num):
return measurer.SnapshotMeasurer(fuzzer, benchmark, trial_num,
SNAPSHOT_LOGGER)
yield _create_measurer
def build_all_measurers(benchmarks: List[str]) -> List[str]:
"""Build measurers for benchmarks."""
logger.info('Building measurers.')
filesystem.recreate_directory(build_utils.get_coverage_binaries_dir())
benchmarks = [(benchmark, ) for benchmark in benchmarks]
results = retry_build_loop(build_measurer, benchmarks)
logger.info('Done building measurers.')
# Return list of benchmarks (like the list we were passed as an argument)
# instead of returning a list of tuples each containing a benchmark.
return [result[0] for result in results]
def set_up_coverage_binaries(pool, experiment):
"""Set up coverage binaries for all benchmarks in |experiment|."""
benchmarks = [
trial.benchmark for trial in db_utils.query(models.Trial).distinct(
models.Trial.benchmark).filter(
models.Trial.experiment == experiment)
]
coverage_binaries_dir = build_utils.get_coverage_binaries_dir()
if not os.path.exists(coverage_binaries_dir):
os.makedirs(coverage_binaries_dir)
pool.map(set_up_coverage_binary, benchmarks)
def build_coverage(benchmark):
"""Build coverage image for benchmark on GCB."""
coverage_binaries_dir = exp_path.filestore(
build_utils.get_coverage_binaries_dir())
substitutions = {
'_GCS_COVERAGE_BINARIES_DIR': coverage_binaries_dir,
'_BENCHMARK': benchmark,
}
config_file = get_build_config_file('coverage.yaml')
config_name = 'benchmark-{benchmark}-coverage'.format(benchmark=benchmark)
_build(config_file, config_name, substitutions)
def build_all_measurers(benchmarks: List[str]) -> List[str]:
"""Build measurers for each benchmark in |benchmarks| in parallel
Returns a list of benchmarks built successfully."""
logger.info('Building measurers.')
filesystem.recreate_directory(build_utils.get_coverage_binaries_dir())
build_measurer_args = [(benchmark, ) for benchmark in benchmarks]
successful_calls = retry_build_loop(build_measurer, build_measurer_args)
logger.info('Done building measurers.')
# Return list of benchmarks (like the list we were passed as an argument)
# instead of returning a list of tuples each containing a benchmark.
return [successful_call[0] for successful_call in successful_calls]
def _build_benchmark_coverage(benchmark: str) -> Tuple[int, str]:
"""Build a coverage build of |benchmark| on GCB."""
coverage_binaries_dir = exp_path.gcs(
build_utils.get_coverage_binaries_dir())
substitutions = {
'_GCS_COVERAGE_BINARIES_DIR': coverage_binaries_dir,
'_BENCHMARK': benchmark,
}
config_file = get_build_config_file('coverage.yaml')
config_name = 'benchmark-{benchmark}-coverage'.format(benchmark=benchmark)
return _build(config_file, config_name, substitutions)
def test_measure_snapshot_coverage( # pylint: disable=too-many-locals
self, mocked_is_cycle_unchanged, db, experiment, tmp_path):
"""Integration test for measure_snapshot_coverage."""
# WORK is set by experiment to a directory that only makes sense in a
# fakefs. A directory containing necessary llvm tools is also added to
# PATH.
llvm_tools_path = get_test_data_path('llvm_tools')
os.environ["PATH"] += os.pathsep + llvm_tools_path
os.environ['WORK'] = str(tmp_path)
mocked_is_cycle_unchanged.return_value = False
# Set up the coverage binary.
benchmark = 'freetype2-2017'
coverage_binary_src = get_test_data_path(
'test_measure_snapshot_coverage', benchmark + '-coverage')
benchmark_cov_binary_dir = os.path.join(
build_utils.get_coverage_binaries_dir(), benchmark)
os.makedirs(benchmark_cov_binary_dir)
coverage_binary_dst_dir = os.path.join(benchmark_cov_binary_dir,
'ftfuzzer')
shutil.copy(coverage_binary_src, coverage_binary_dst_dir)
# Set up entities in database so that the snapshot can be created.
experiment = models.Experiment(name=os.environ['EXPERIMENT'])
db_utils.add_all([experiment])
trial = models.Trial(fuzzer=FUZZER,
benchmark=benchmark,
experiment=os.environ['EXPERIMENT'])
db_utils.add_all([trial])
snapshot_measurer = measurer.SnapshotMeasurer(trial.fuzzer,
trial.benchmark,
trial.id,
SNAPSHOT_LOGGER)
# Set up the snapshot archive.
cycle = 1
archive = get_test_data_path('test_measure_snapshot_coverage',
'corpus-archive-%04d.tar.gz' % cycle)
corpus_dir = os.path.join(snapshot_measurer.trial_dir, 'corpus')
os.makedirs(corpus_dir)
shutil.copy(archive, corpus_dir)
with mock.patch('common.filestore_utils.cp') as mocked_cp:
mocked_cp.return_value = new_process.ProcessResult(0, '', False)
# TODO(metzman): Create a system for using actual buckets in
# integration tests.
snapshot = measurer.measure_snapshot_coverage(
snapshot_measurer.fuzzer, snapshot_measurer.benchmark,
snapshot_measurer.trial_num, cycle)
assert snapshot
assert snapshot.time == cycle * experiment_utils.get_snapshot_seconds()
assert snapshot.edges_covered == 13178
def _build_oss_fuzz_project_coverage(benchmark: str) -> Tuple[int, str]:
"""Build a coverage build of OSS-Fuzz-based benchmark |benchmark| on GCB."""
coverage_binaries_dir = exp_path.filestore(
build_utils.get_coverage_binaries_dir())
substitutions = {
'_GCS_COVERAGE_BINARIES_DIR': coverage_binaries_dir,
'_BENCHMARK': benchmark,
}
config_file = get_build_config_file('oss-fuzz-coverage.yaml')
config_name = 'oss-fuzz-{benchmark}-coverage'.format(benchmark=benchmark)
return _build(config_file, config_name, substitutions)
def set_up_coverage_binaries(pool, experiment):
"""Set up coverage binaries for all benchmarks in |experiment|."""
# Use set comprehension to select distinct benchmarks.
benchmarks = [
benchmark_tuple[0]
for benchmark_tuple in db_utils.query(models.Trial.benchmark).distinct(
).filter(models.Trial.experiment == experiment)
]
coverage_binaries_dir = build_utils.get_coverage_binaries_dir()
filesystem.create_directory(coverage_binaries_dir)
pool.map(set_up_coverage_binary, benchmarks)
def set_up_coverage_binary(benchmark):
"""Set up coverage binaries for |benchmark|."""
initialize_logs()
coverage_binaries_dir = build_utils.get_coverage_binaries_dir()
benchmark_coverage_binary_dir = coverage_binaries_dir / benchmark
filesystem.create_directory(benchmark_coverage_binary_dir)
archive_name = 'coverage-build-%s.tar.gz' % benchmark
archive_filestore_path = exp_path.filestore(coverage_binaries_dir /
archive_name)
filestore_utils.cp(archive_filestore_path,
str(benchmark_coverage_binary_dir))
archive_path = benchmark_coverage_binary_dir / archive_name
tar = tarfile.open(archive_path, 'r:gz')
tar.extractall(benchmark_coverage_binary_dir)
os.remove(archive_path)
def _build_oss_fuzz_project_coverage(benchmark: str) -> Tuple[int, str]:
"""Build a coverage build of OSS-Fuzz-based benchmark |benchmark| on GCB."""
project = benchmark_utils.get_project(benchmark)
oss_fuzz_builder_hash = benchmark_utils.get_oss_fuzz_builder_hash(benchmark)
coverage_binaries_dir = exp_path.gcs(
build_utils.get_coverage_binaries_dir())
substitutions = {
'_GCS_COVERAGE_BINARIES_DIR': coverage_binaries_dir,
'_OSS_FUZZ_PROJECT': project,
'_OSS_FUZZ_BUILDER_HASH': oss_fuzz_builder_hash,
}
config_file = get_build_config_file('oss-fuzz-coverage.yaml')
config_name = 'oss-fuzz-{project}-coverage-hash-{hash}'.format(
project=project, hash=oss_fuzz_builder_hash)
return _build(config_file, config_name, substitutions)
def test_measure_snapshot_coverage( # pylint: disable=too-many-locals
self, mocked_is_cycle_unchanged, create_measurer, db, experiment):
"""Integration test for measure_snapshot_coverage."""
mocked_is_cycle_unchanged.return_value = False
# Set up the coverage binary.
benchmark = 'freetype2-2017'
coverage_binary_src = get_test_data_path(
'test_measure_snapshot_coverage', benchmark + '-coverage')
benchmark_cov_binary_dir = os.path.join(
build_utils.get_coverage_binaries_dir(), benchmark)
os.makedirs(benchmark_cov_binary_dir)
coverage_binary_dst_dir = os.path.join(benchmark_cov_binary_dir,
'fuzz-target')
shutil.copy(coverage_binary_src, coverage_binary_dst_dir)
# Set up entities in database so that the snapshot can be created.
experiment = models.Experiment(name=os.environ['EXPERIMENT'])
db_utils.add_all([experiment])
trial = models.Trial(fuzzer=FUZZER,
benchmark=benchmark,
experiment=os.environ['EXPERIMENT'])
db_utils.add_all([trial])
snapshot_measurer = create_measurer(trial.fuzzer, trial.benchmark,
trial.id)
# Set up the snapshot archive.
cycle = 1
archive = get_test_data_path('test_measure_snapshot_coverage',
'corpus-archive-%04d.tar.gz' % cycle)
corpus_dir = os.path.join(snapshot_measurer.trial_dir, 'corpus')
os.makedirs(corpus_dir)
shutil.copy(archive, corpus_dir)
with mock.patch('common.gsutil.cp') as mocked_cp:
mocked_cp.return_value = new_process.ProcessResult(0, '', False)
# TODO(metzman): Create a system for using actual buckets in
# integration tests.
snapshot = measurer.measure_snapshot_coverage(
snapshot_measurer.fuzzer, snapshot_measurer.benchmark,
snapshot_measurer.trial_num, cycle)
assert snapshot
assert snapshot.time == cycle * experiment_utils.get_snapshot_seconds()
assert snapshot.edges_covered == 3798
def set_up_coverage_binary(benchmark):
"""Set up coverage binaries for |benchmark|."""
initialize_logs()
coverage_binaries_dir = build_utils.get_coverage_binaries_dir()
benchmark_coverage_binary_dir = coverage_binaries_dir / benchmark
if not os.path.exists(benchmark_coverage_binary_dir):
os.mkdir(benchmark_coverage_binary_dir)
archive_name = 'coverage-build-%s.tar.gz' % benchmark
cloud_bucket_archive_path = exp_path.gcs(coverage_binaries_dir /
archive_name)
gsutil.cp(cloud_bucket_archive_path,
str(benchmark_coverage_binary_dir),
write_to_stdout=False)
archive_path = benchmark_coverage_binary_dir / archive_name
tar = tarfile.open(archive_path, 'r:gz')
tar.extractall(benchmark_coverage_binary_dir)
os.remove(archive_path)
def copy_coverage_binaries(benchmark):
"""Copy coverage binaries in a local experiment."""
shared_coverage_binaries_dir = get_shared_coverage_binaries_dir()
mount_arg = '{0}:{0}'.format(shared_coverage_binaries_dir)
builder_image_url = benchmark_utils.get_builder_image_url(
benchmark, 'coverage', environment.get('CLOUD_PROJECT'))
coverage_build_archive = 'coverage-build-{}.tar.gz'.format(benchmark)
coverage_build_archive_shared_dir_path = os.path.join(
shared_coverage_binaries_dir, coverage_build_archive)
command = 'cd /out; tar -czvf {} *'.format(
coverage_build_archive_shared_dir_path)
new_process.execute([
'docker', 'run', '-v', mount_arg, builder_image_url, '/bin/bash', '-c',
command
])
coverage_binaries_dir = build_utils.get_coverage_binaries_dir()
coverage_build_archive_gcs_path = posixpath.join(
exp_path.gcs(coverage_binaries_dir), coverage_build_archive)
return gsutil.cp(coverage_build_archive_shared_dir_path,
coverage_build_archive_gcs_path)