def update_environment(env):
"""Update worker's environment."""
processed_env = {}
for key, value in env.iteritems():
if should_rebase_environment_value(key):
value = file_host.rebase_to_worker_root(value)
processed_env[key] = value
request = untrusted_runner_pb2.UpdateEnvironmentRequest(env=processed_env)
host.stub().UpdateEnvironment(request)
def list_files(path, recursive=False):
"""List files in the directory. Returns full file paths."""
request = untrusted_runner_pb2.ListFilesRequest(path=path,
recursive=recursive)
response = host.stub().ListFiles(request)
return response.file_paths
def create_directory(path, create_intermediates=False):
"""Create a directory."""
request = untrusted_runner_pb2.CreateDirectoryRequest(
path=path, create_intermediates=create_intermediates)
response = host.stub().CreateDirectory(request)
return response.result
def run_process(cmdline,
current_working_directory=None,
timeout=process_handler.DEFAULT_TEST_TIMEOUT,
need_shell=False,
gestures=None,
env_copy=None,
testcase_run=True,
ignore_children=True):
"""Remote version of process_handler.run_process."""
request = untrusted_runner_pb2.RunProcessRequest(
cmdline=cmdline,
current_working_directory=current_working_directory,
timeout=timeout,
need_shell=need_shell,
testcase_run=testcase_run,
ignore_children=ignore_children)
if gestures:
request.gestures.extend(gestures)
env = {}
# run_process's local behaviour is to apply the passed |env_copy| on top of
# the current environment instead of replacing it completely (like with
# subprocess).
environment.set_environment_vars(env, os.environ)
environment.set_environment_vars(env, env_copy)
request.env_copy.update(env)
response = host.stub().RunProcess(request)
return response.return_code, response.execution_time, response.output
def write_data_to_worker(data, worker_path):
"""Write data to a file on the worker."""
request_iterator = file_utils.data_chunk_generator(data)
metadata = [('path-bin', worker_path.encode('utf-8'))]
response = host.stub().CopyFileTo(request_iterator, metadata=metadata)
return response.result
def remove_directory(path, recreate=False):
"""Remove a directory. If |recreate| is set, always creates the directory even
if it did not exist."""
request = untrusted_runner_pb2.RemoveDirectoryRequest(path=path,
recreate=recreate)
response = host.stub().RemoveDirectory(request)
return response.result
def stat(path):
"""stat() a path."""
request = untrusted_runner_pb2.StatRequest(path=path)
response = host.stub().Stat(request)
if not response.result:
return None
return response
def setup(self):
request = untrusted_runner_pb2.SetupProductionBuildRequest(
base_build_dir=self.base_build_dir,
version=self.revision,
build_url=self.build_url,
build_type=self.build_type)
return _handle_response(self, host.stub().SetupProductionBuild(request))
def setup(self):
request = untrusted_runner_pb2.SetupSymbolizedBuildRequest(
base_build_dir=self.base_build_dir,
revision=self.revision,
release_build_url=self.release_build_url,
debug_build_url=self.debug_build_url)
return _handle_response(self, host.stub().SetupSymbolizedBuild(request))
def copy_file_to_worker(host_path, worker_path):
"""Copy file from host to worker. |worker_path| must be a full path (including
the filename). Any directories will be created if needed."""
with open(host_path) as f:
request_iterator = file_utils.file_chunk_generator(f)
metadata = [('path-bin', worker_path.encode('utf-8'))]
response = host.stub().CopyFileTo(request_iterator, metadata=metadata)
return response.result
def setup(self):
request = untrusted_runner_pb2.SetupRegularBuildRequest(
base_build_dir=self.base_build_dir,
revision=self.revision,
build_url=self.build_url,
build_dir_name=self.build_dir_name)
if self.target_weights:
request.target_weights.update(self.target_weights)
return _handle_response(self, host.stub().SetupRegularBuild(request))
def symbolize_stacktrace(unsymbolized_crash_stacktrace,
enable_inline_frames=True):
"""Symbolize stacktrace."""
request = untrusted_runner_pb2.SymbolizeStacktraceRequest(
unsymbolized_crash_stacktrace=protobuf_utils.encode_utf8_if_unicode(
unsymbolized_crash_stacktrace),
enable_inline_frames=enable_inline_frames)
response = host.stub().SymbolizeStacktrace(request)
return response.symbolized_stacktrace
def do_corpus_pruning(context, last_execution_failed, revision):
"""Do corpus pruning on untrusted worker."""
cross_pollinate_fuzzers = [
untrusted_runner_pb2.CrossPollinateFuzzer(
fuzz_target=_fuzz_target_to_proto(cpf.fuzz_target),
backup_bucket_name=cpf.backup_bucket_name,
corpus_engine_name=cpf.corpus_engine_name,
) for cpf in context.cross_pollinate_fuzzers
]
request = untrusted_runner_pb2.PruneCorpusRequest(
fuzz_target=_fuzz_target_to_proto(context.fuzz_target),
cross_pollinate_fuzzers=cross_pollinate_fuzzers,
last_execution_failed=last_execution_failed,
revision=revision)
response = host.stub().PruneCorpus(request)
project_qualified_name = context.fuzz_target.project_qualified_name()
today_date = datetime.datetime.utcnow().date()
coverage_info = data_types.CoverageInformation(
fuzzer=project_qualified_name, date=today_date)
# Intentionally skip edge and function coverage values as those would come
# from fuzzer coverage cron task (see src/go/server/cron/coverage.go).
coverage_info.corpus_size_units = response.coverage_info.corpus_size_units
coverage_info.corpus_size_bytes = response.coverage_info.corpus_size_bytes
coverage_info.corpus_location = response.coverage_info.corpus_location
coverage_info.corpus_backup_location = (
response.coverage_info.corpus_backup_location)
coverage_info.quarantine_size_units = (
response.coverage_info.quarantine_size_units)
coverage_info.quarantine_size_bytes = (
response.coverage_info.quarantine_size_bytes)
coverage_info.quarantine_location = response.coverage_info.quarantine_location
crashes = [
corpus_pruning_task.CorpusCrash(
crash_state=crash.crash_state,
crash_type=crash.crash_type,
crash_address=crash.crash_address,
crash_stacktrace=utils.decode_to_unicode(crash.crash_stacktrace),
unit_path=crash.unit_path,
security_flag=crash.security_flag,
) for crash in response.crashes
]
return corpus_pruning_task.CorpusPruningResult(
coverage_info=coverage_info,
crashes=crashes,
fuzzer_binary_name=response.fuzzer_binary_name,
revision=response.revision)
def copy_file_from_worker(worker_path, host_path):
"""Copy file from worker to host."""
request = untrusted_runner_pb2.CopyFileFromRequest(path=worker_path)
response = host.stub().CopyFileFrom(request)
file_utils.write_chunks(host_path, response)
metadata = dict(response.trailing_metadata())
if metadata.get('result') != 'ok':
# file_utils.write_chunks always opens the file for writing, so remove it
# here.
os.remove(host_path)
return False
return True
def run_and_wait(self,
additional_args=None,
timeout=None,
terminate_before_kill=False,
terminate_wait_time=None,
input_data=None,
max_stdout_len=None,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
**popen_args):
"""Remote version of new_process.ProcessRunner.run_and_wait."""
assert stdout == subprocess.PIPE
assert stderr == subprocess.STDOUT
request = untrusted_runner_pb2.RunAndWaitRequest(
executable_path=self.executable_path,
timeout=timeout,
terminate_before_kill=terminate_before_kill,
terminate_wait_time=terminate_wait_time,
input_data=input_data,
max_stdout_len=max_stdout_len)
request.default_args.extend(self.default_args)
request.additional_args.extend(additional_args)
if 'bufsize' in popen_args:
request.popen_args.bufsize = popen_args['bufsize']
if 'executable' in popen_args:
request.popen_args.executable = popen_args['executable']
if 'shell' in popen_args:
request.popen_args.shell = popen_args['shell']
if 'cwd' in popen_args:
request.popen_args.cwd = popen_args['cwd']
passed_env = popen_args.get('env', None)
if passed_env is not None:
request.popen_args.env_is_set = True
# Filter the passed environment to prevent leaking sensitive environment
# variables if the caller passes e.g. os.environ.copy().
environment.set_environment_vars(request.popen_args.env,
passed_env)
response = host.stub().RunAndWait(request)
return process_result_from_proto(response.result)
def reset_environment():
"""Reset environment variables."""
request = untrusted_runner_pb2.ResetEnvironmentRequest()
host.stub().ResetEnvironment(request)
def get_fuzz_targets(path):
"""Get list of fuzz target paths."""
request = untrusted_runner_pb2.GetFuzzTargetsRequest(path=path)
response = host.stub().GetFuzzTargets(request)
return response.fuzz_target_paths
def terminate_stale_application_instances():
"""Terminate stale application instances."""
host.stub().TerminateStaleApplicationInstances(
untrusted_runner_pb2.TerminateStaleApplicationInstancesRequest())