def _build_benchmark_fuzzer(benchmark: str, fuzzer: str) -> Tuple[int, str]:
"""Build a |benchmark|, |fuzzer| runner image on GCB."""
underlying_fuzzer = fuzzer_config_utils.get_by_variant_name(
fuzzer)['fuzzer']
# See link for why substitutions must begin with an underscore:
# https://cloud.google.com/cloud-build/docs/configuring-builds/substitute-variable-values#using_user-defined_substitutions
substitutions = {
'_BENCHMARK': benchmark,
'_FUZZER': underlying_fuzzer,
}
config_file = get_build_config_file('fuzzer.yaml')
config_name = 'benchmark-{benchmark}-fuzzer-{fuzzer}'.format(
benchmark=benchmark, fuzzer=fuzzer)
return _build(config_file, config_name, substitutions)
def _add_build_arguments_to_config(base: str, fuzzer: str) -> str:
"""If there are fuzzer-specific arguments, make a config file with them."""
fuzzer_config = fuzzer_config_utils.get_by_variant_name(fuzzer)
if 'build_arguments' not in fuzzer_config:
return base
# TODO(mbarbella): Rather than rewrite yaml files, use the GCB API.
args = fuzzer_config['build_arguments']
config = yaml_utils.read(base)
for step in config['steps']:
if 'id' in step and step['id'] in BUILDER_STEP_IDS:
# Append additional flags before the final argument.
step['args'] = step['args'][:-1] + args + [step['args'][-1]]
new_config_path = os.path.join(CONFIG_DIR, 'builds', fuzzer + '.yaml')
filesystem.create_directory(os.path.dirname(new_config_path))
yaml_utils.write(new_config_path, config)
return new_config_path
def _build_oss_fuzz_project_fuzzer(benchmark: str,
fuzzer: str) -> Tuple[int, str]:
"""Build a |benchmark|, |fuzzer| runner image on GCB."""
underlying_fuzzer = fuzzer_config_utils.get_by_variant_name(
fuzzer)['fuzzer']
project = benchmark_utils.get_project(benchmark)
oss_fuzz_builder_hash = benchmark_utils.get_oss_fuzz_builder_hash(
benchmark)
substitutions = {
'_OSS_FUZZ_PROJECT': project,
'_BENCHMARK': benchmark,
'_FUZZER': underlying_fuzzer,
'_OSS_FUZZ_BUILDER_HASH': oss_fuzz_builder_hash,
}
config_file = get_build_config_file('oss-fuzz-fuzzer.yaml')
config_name = 'oss-fuzz-{project}-fuzzer-{fuzzer}-hash-{hash}'.format(
project=project, fuzzer=fuzzer, hash=oss_fuzz_builder_hash)
return _build(config_file, config_name, substitutions)
def render_startup_script_template(instance_name: str, benchmark: str,
fuzzer: str, trial_id: int,
experiment_config: dict):
"""Render the startup script using the template and the parameters
provided and return the result."""
fuzzer_config = fuzzer_config_utils.get_by_variant_name(fuzzer)
underlying_fuzzer_name = fuzzer_config['fuzzer']
docker_image_url = benchmark_utils.get_runner_image_url(
benchmark, underlying_fuzzer_name, experiment_config['cloud_project'])
fuzz_target = benchmark_utils.get_fuzz_target(benchmark)
# Convert additional environment variables from configuration to arguments
# that will be passed to docker.
additional_env = ''
if 'env' in fuzzer_config:
additional_env = ' '.join([
'-e {k}={v}'.format(k=k, v=shlex.quote(v))
for k, v in fuzzer_config['env'].items()
])
local_experiment = experiment_utils.is_local_experiment()
template = JINJA_ENV.get_template('runner-startup-script-template.sh')
kwargs = {
'instance_name': instance_name,
'benchmark': benchmark,
'experiment': experiment_config['experiment'],
'fuzzer': underlying_fuzzer_name,
'fuzzer_variant_name': fuzzer,
'trial_id': trial_id,
'max_total_time': experiment_config['max_total_time'],
'cloud_project': experiment_config['cloud_project'],
'cloud_compute_zone': experiment_config['cloud_compute_zone'],
'cloud_experiment_bucket':
experiment_config['cloud_experiment_bucket'],
'fuzz_target': fuzz_target,
'docker_image_url': docker_image_url,
'additional_env': additional_env,
'local_experiment': local_experiment
}
if local_experiment:
kwargs['host_gcloud_config'] = os.environ['HOST_GCLOUD_CONFIG']
return template.render(**kwargs)
def create_trial_instance(benchmark: str, fuzzer: str, trial_id: int,
experiment_config: dict) -> bool:
"""Create or start a trial instance for a specific
trial_id,fuzzer,benchmark."""
instance_name = experiment_utils.get_trial_instance_name(
experiment_config['experiment'], trial_id)
fuzzer_config = fuzzer_config_utils.get_by_variant_name(fuzzer)
underlying_fuzzer_name = fuzzer_config['fuzzer']
docker_image_url = benchmark_utils.get_runner_image_url(
benchmark, underlying_fuzzer_name, experiment_config['cloud_project'])
fuzz_target = benchmark_utils.get_fuzz_target(benchmark)
# Convert additional environment variables from configuration to arguments
# that will be passed to docker.
additional_env = ''
if 'env' in fuzzer_config:
additional_env = ' '.join([
'-e {k}={v}'.format(k=k, v=shlex.quote(v))
for k, v in fuzzer_config['env'].items()
])
startup_script = '''#!/bin/bash
echo 0 > /proc/sys/kernel/yama/ptrace_scope
echo core >/proc/sys/kernel/core_pattern
while ! docker pull {docker_image_url}
do
echo 'Error pulling image, retrying...'
done
docker run --privileged --cpuset-cpus=0 --rm \
-e INSTANCE_NAME={instance_name} -e FUZZER={fuzzer} -e BENCHMARK={benchmark} \
-e FUZZER_VARIANT_NAME={fuzzer_variant_name} -e EXPERIMENT={experiment} \
-e TRIAL_ID={trial_id} -e MAX_TOTAL_TIME={max_total_time} \
-e CLOUD_PROJECT={cloud_project} -e CLOUD_COMPUTE_ZONE={cloud_compute_zone} \
-e CLOUD_EXPERIMENT_BUCKET={cloud_experiment_bucket} \
-e FUZZ_TARGET={fuzz_target} {additional_env} \
--cap-add SYS_NICE --cap-add SYS_PTRACE --name=runner-container \
{docker_image_url} 2>&1 | tee /tmp/runner-log.txt'''.format(
instance_name=instance_name,
benchmark=benchmark,
experiment=experiment_config['experiment'],
fuzzer=underlying_fuzzer_name,
fuzzer_variant_name=fuzzer,
trial_id=trial_id,
max_total_time=experiment_config['max_total_time'],
cloud_project=experiment_config['cloud_project'],
cloud_compute_zone=experiment_config['cloud_compute_zone'],
cloud_experiment_bucket=experiment_config['cloud_experiment_bucket'],
fuzz_target=fuzz_target,
docker_image_url=docker_image_url,
additional_env=additional_env)
startup_script_path = '/tmp/%s-start-docker.sh' % instance_name
with open(startup_script_path, 'w') as file_handle:
file_handle.write(startup_script)
return gcloud.create_instance(instance_name,
gcloud.InstanceType.RUNNER,
experiment_config,
startup_script=startup_script_path,
write_to_stdout=False)