def get_experiment_tag_for_image(image_specs, tag_by_experiment=True):
"""Returns the registry with the experiment tag for given image."""
tag = posixpath.join(experiment_utils.get_base_docker_tag(),
image_specs['tag'])
if tag_by_experiment:
tag += ':' + experiment_utils.get_experiment_name()
return tag
def get_builder_image_url(benchmark, fuzzer, cloud_project):
"""Get the URL of the docker builder image for fuzzing the benchmark with
fuzzer."""
base_tag = experiment_utils.get_base_docker_tag(cloud_project)
if is_oss_fuzz(benchmark):
return '{base_tag}/oss-fuzz/builders/{fuzzer}/{project}'.format(
base_tag=base_tag, fuzzer=fuzzer, project=get_project(benchmark))
return '{base_tag}/builders/{fuzzer}/{benchmark}'.format(
base_tag=base_tag, fuzzer=fuzzer, benchmark=benchmark)
def start(self):
"""Start the experiment on the dispatcher."""
# TODO(metzman): Replace this workflow with a startup script so we don't
# need to SSH into the dispatcher.
self.process.join() # Wait for dispatcher instance.
# Check that we can SSH into the instance.
gcloud.robust_begin_gcloud_ssh(self.instance_name,
self.config['cloud_compute_zone'])
base_docker_tag = experiment_utils.get_base_docker_tag(
self.config['cloud_project'])
cloud_sql_instance_connection_name = (
self.config['cloud_sql_instance_connection_name'])
command = (
'echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope && '
'docker run --rm '
'-e INSTANCE_NAME="{instance_name}" '
'-e EXPERIMENT="{experiment}" '
'-e CLOUD_PROJECT="{cloud_project}" '
'-e CLOUD_EXPERIMENT_BUCKET="{cloud_experiment_bucket}" '
'-e POSTGRES_PASSWORD="******" '
'-e CLOUD_SQL_INSTANCE_CONNECTION_NAME='
'"{cloud_sql_instance_connection_name}" '
'--cap-add=SYS_PTRACE --cap-add=SYS_NICE '
'-v /var/run/docker.sock:/var/run/docker.sock '
'--name=dispatcher-container '
'{base_docker_tag}/dispatcher-image '
'/work/startup-dispatcher.sh'
).format(
instance_name=self.instance_name,
postgres_password=os.environ['POSTGRES_PASSWORD'],
experiment=self.config['experiment'],
# TODO(metzman): Create a function that sets env vars based on
# the contents of a dictionary, and use it instead of hardcoding
# the configs we use.
cloud_project=self.config['cloud_project'],
cloud_experiment_bucket=self.config['cloud_experiment_bucket'],
cloud_sql_instance_connection_name=(
cloud_sql_instance_connection_name),
base_docker_tag=base_docker_tag,
)
return gcloud.ssh(self.instance_name,
command=command,
zone=self.config['cloud_compute_zone'])
def _build(
config_file: str,
config_name: str,
substitutions: Dict[str, str] = None,
timeout_seconds: int = GCB_BUILD_TIMEOUT) -> new_process.ProcessResult:
"""Build each of |args| on gcb."""
config_arg = '--config=%s' % config_file
machine_type_arg = '--machine-type=%s' % GCB_MACHINE_TYPE
# Use "s" suffix to denote seconds.
timeout_arg = '--timeout=%ds' % timeout_seconds
command = [
'gcloud',
'builds',
'submit',
str(utils.ROOT_DIR),
config_arg,
timeout_arg,
machine_type_arg,
]
if substitutions is None:
substitutions = {}
assert '_REPO' not in substitutions
substitutions['_REPO'] = experiment_utils.get_base_docker_tag()
assert '_EXPERIMENT' not in substitutions
substitutions['_EXPERIMENT'] = experiment_utils.get_experiment_name()
substitutions = [
'%s=%s' % (key, value) for key, value in substitutions.items()
]
substitutions = ','.join(substitutions)
command.append('--substitutions=%s' % substitutions)
# Don't write to stdout to make concurrent building faster. Otherwise
# writing becomes the bottleneck.
result = new_process.execute(command,
write_to_stdout=False,
kill_children=True,
timeout=timeout_seconds)
build_utils.store_build_logs(config_name, result)
return result
def coverage_steps(benchmark):
"""Returns GCB run steps for coverage builds."""
coverage_binaries_dir = exp_path.filestore(
build_utils.get_coverage_binaries_dir())
steps = [{
'name':
DOCKER_IMAGE,
'args': [
'run', '-v', '/workspace/out:/host-out',
posixpath.join(experiment_utils.get_base_docker_tag(), 'builders',
'coverage', benchmark) + ':' +
experiment_utils.get_experiment_name(), '/bin/bash', '-c',
'cd /out; tar -czvf /host-out/coverage-build-' + benchmark +
'.tar.gz * /src /work'
]
}]
step = {'name': 'gcr.io/cloud-builders/gsutil'}
step['args'] = [
'-m', 'cp', '/workspace/out/coverage-build-' + benchmark + '.tar.gz',
coverage_binaries_dir + '/'
]
steps.append(step)
return steps
def start(self):
"""Start the experiment on the dispatcher."""
shared_volume_dir = os.path.abspath('shared-volume')
if not os.path.exists(shared_volume_dir):
os.mkdir(shared_volume_dir)
shared_volume_volume_arg = '{0}:{0}'.format(shared_volume_dir)
shared_volume_env_arg = 'SHARED_VOLUME={}'.format(shared_volume_dir)
sql_database_arg = 'SQL_DATABASE_URL=sqlite:///{}'.format(
os.path.join(shared_volume_dir, 'local.db'))
home = os.environ['HOME']
host_gcloud_config_arg = (
'HOST_GCLOUD_CONFIG={home}/{gcloud_config_dir}'.format(
home=home, gcloud_config_dir='.config/gcloud'))
base_docker_tag = experiment_utils.get_base_docker_tag(
self.config['cloud_project'])
set_instance_name_arg = 'INSTANCE_NAME={instance_name}'.format(
instance_name=self.instance_name)
set_experiment_arg = 'EXPERIMENT={experiment}'.format(
experiment=self.config['experiment'])
set_cloud_project_arg = 'CLOUD_PROJECT={cloud_project}'.format(
cloud_project=self.config['cloud_project'])
set_cloud_experiment_bucket_arg = (
'CLOUD_EXPERIMENT_BUCKET={cloud_experiment_bucket}'.format(
cloud_experiment_bucket=self.config['cloud_experiment_bucket']))
docker_image_url = '{base_docker_tag}/dispatcher-image'.format(
base_docker_tag=base_docker_tag)
volume_arg = '{home}/.config/gcloud:/root/.config/gcloud'.format(
home=home)
command = [
'docker',
'run',
'-ti',
'--rm',
'-v',
volume_arg,
'-v',
'/var/run/docker.sock:/var/run/docker.sock',
'-v',
shared_volume_volume_arg,
'-e',
shared_volume_env_arg,
'-e',
host_gcloud_config_arg,
'-e',
set_instance_name_arg,
'-e',
set_experiment_arg,
'-e',
set_cloud_project_arg,
'-e',
sql_database_arg,
'-e',
set_cloud_experiment_bucket_arg,
'-e',
'LOCAL_EXPERIMENT=True',
'--cap-add=SYS_PTRACE',
'--cap-add=SYS_NICE',
'--name=dispatcher-container',
docker_image_url,
'/bin/bash',
'-c',
'gsutil -m rsync -r '
'"${CLOUD_EXPERIMENT_BUCKET}/${EXPERIMENT}/input" ${WORK} && '
'source "/work/.venv/bin/activate" && '
'pip3 install -r "/work/src/requirements.txt" && '
'PYTHONPATH=/work/src python3 '
'/work/src/experiment/dispatcher.py || '
'/bin/bash' # Open shell if experiment fails.
]
return new_process.execute(command, write_to_stdout=True)
def get_runner_image_url(benchmark, fuzzer, cloud_project):
"""Get the URL of the docker runner image for fuzzing the benchmark with
fuzzer."""
base_tag = experiment_utils.get_base_docker_tag(cloud_project)
return '{base_tag}/runners/{fuzzer}/{benchmark}'.format(
base_tag=base_tag, fuzzer=fuzzer, benchmark=benchmark)
def start(self):
"""Start the experiment on the dispatcher."""
shared_volume_dir = os.path.abspath('shared-volume')
if not os.path.exists(shared_volume_dir):
os.mkdir(shared_volume_dir)
shared_volume_volume_arg = '{0}:{0}'.format(shared_volume_dir)
shared_volume_env_arg = 'SHARED_VOLUME={}'.format(shared_volume_dir)
sql_database_arg = 'SQL_DATABASE_URL=sqlite:///{}'.format(
os.path.join(shared_volume_dir, 'local.db'))
base_docker_tag = experiment_utils.get_base_docker_tag(
self.config['cloud_project'])
set_instance_name_arg = 'INSTANCE_NAME={instance_name}'.format(
instance_name=self.instance_name)
set_experiment_arg = 'EXPERIMENT={experiment}'.format(
experiment=self.config['experiment'])
set_cloud_project_arg = 'CLOUD_PROJECT={cloud_project}'.format(
cloud_project=self.config['cloud_project'])
shared_experiment_filestore_arg = '{0}:{0}'.format(
self.config['experiment_filestore'])
set_experiment_filestore_arg = (
'EXPERIMENT_FILESTORE={experiment_filestore}'.format(
experiment_filestore=self.config['experiment_filestore']))
shared_report_filestore_arg = '{0}:{0}'.format(
self.config['report_filestore'])
set_report_filestore_arg = (
'REPORT_FILESTORE={report_filestore}'.format(
report_filestore=self.config['report_filestore']))
docker_image_url = '{base_docker_tag}/dispatcher-image'.format(
base_docker_tag=base_docker_tag)
command = [
'docker',
'run',
'-ti',
'--rm',
'-v',
'/var/run/docker.sock:/var/run/docker.sock',
'-v',
shared_volume_volume_arg,
'-v',
shared_experiment_filestore_arg,
'-v',
shared_report_filestore_arg,
'-e',
shared_volume_env_arg,
'-e',
set_instance_name_arg,
'-e',
set_experiment_arg,
'-e',
set_cloud_project_arg,
'-e',
sql_database_arg,
'-e',
set_experiment_filestore_arg,
'-e',
set_report_filestore_arg,
'-e',
'LOCAL_EXPERIMENT=True',
'--cap-add=SYS_PTRACE',
'--cap-add=SYS_NICE',
'--name=dispatcher-container',
docker_image_url,
'/bin/bash',
'-c',
'rsync -r '
'"${EXPERIMENT_FILESTORE}/${EXPERIMENT}/input/" ${WORK} && '
'mkdir ${WORK}/src && '
'tar -xvzf ${WORK}/src.tar.gz -C ${WORK}/src && '
'source "${WORK}/.venv/bin/activate" && '
'pip3 install -r "${WORK}/src/requirements.txt" && '
'PYTHONPATH=${WORK}/src python3 '
'${WORK}/src/experiment/dispatcher.py || '
'/bin/bash' # Open shell if experiment fails.
]
return new_process.execute(command, write_to_stdout=True)
