def process_testcase(engine_name, tool_name, target_name, arguments,
testcase_path, output_path, timeout):
"""Process testcase on untrusted worker."""
if tool_name == 'minimize':
operation = untrusted_runner_pb2.ProcessTestcaseRequest.MINIMIZE
else:
operation = untrusted_runner_pb2.ProcessTestcaseRequest.CLEANSE
rebased_testcase_path = file_host.rebase_to_worker_root(testcase_path)
file_host.copy_file_to_worker(testcase_path, rebased_testcase_path)
request = untrusted_runner_pb2.ProcessTestcaseRequest(
engine=engine_name,
operation=operation,
target_name=target_name,
arguments=arguments,
testcase_path=file_host.rebase_to_worker_root(testcase_path),
output_path=file_host.rebase_to_worker_root(output_path),
timeout=timeout)
response = host.stub().ProcessTestcase(request)
rebased_output_path = file_host.rebase_to_worker_root(output_path)
file_host.copy_file_from_worker(rebased_output_path, output_path)
return engine.ReproduceResult(list(response.command), response.return_code,
response.time_executed, response.output)
def get_file_from_untrusted_worker(worker_file_path):
"""Gets file from an untrusted worker to local. Local file stays in the temp
folder until the end of task or can be explicitly deleted by the caller."""
from bot.untrusted_runner import file_host
with tempfile.NamedTemporaryFile(delete=False, dir=get_temp_dir()) as f:
local_file_path = f.name
file_host.copy_file_from_worker(worker_file_path, local_file_path)
return local_file_path
def test_copy_file_from_worker_failure(self):
"""Test file_host.copy_file_from_worker (failure)."""
mock_response = mock.MagicMock()
mock_response.trailing_metadata.return_value = (('result', 'invalid-path'),)
self.mock.stub().CopyFileFrom.return_value = mock_response
self.assertFalse(file_host.copy_file_from_worker('/file', '/file'))
self.assertFalse(os.path.exists('/file'))
def test_copy_file_from_worker(self):
"""Tests remote copy_file_from_worker."""
src_path = os.path.join(self.tmp_dir, 'src')
with open(src_path, 'w') as f:
f.write(TEST_FILE_CONTENTS)
dest_path = os.path.join(self.tmp_dir, 'dst')
self.assertTrue(file_host.copy_file_from_worker(src_path, dest_path))
with open(dest_path) as f:
self.assertEqual(f.read(), TEST_FILE_CONTENTS)
def _is_data_bundle_up_to_date(data_bundle, data_bundle_directory):
"""Return true if the data bundle is up to date, false otherwise."""
sync_file_path = _get_data_bundle_sync_file_path(data_bundle_directory)
if environment.is_trusted_host() and data_bundle.sync_to_worker:
from bot.untrusted_runner import file_host
worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path)
shell.remove_file(sync_file_path)
file_host.copy_file_from_worker(worker_sync_file_path, sync_file_path)
if not os.path.exists(sync_file_path):
return False
last_sync_time = datetime.datetime.utcfromtimestamp(
utils.read_data_from_file(sync_file_path))
# Check if we recently synced.
if not dates.time_has_expired(
last_sync_time, seconds=_DATA_BUNDLE_SYNC_INTERVAL_IN_SECONDS):
return True
# For search index data bundle, we don't sync them from bucket. Instead, we
# rely on the fuzzer to generate testcases periodically.
if _is_search_index_data_bundle(data_bundle.name):
return False
# Check when the bucket url had last updates. If no new updates, no need to
# update directory.
bucket_url = data_handler.get_data_bundle_bucket_url(data_bundle.name)
last_updated_time = storage.last_updated(bucket_url)
if last_updated_time and last_sync_time > last_updated_time:
logs.log("Data bundle %s has no new content from last sync." %
data_bundle.name)
return True
return False
def test_copy_file_from_worker(self):
"""Test file_host.copy_file_from_worker."""
mock_response = mock.MagicMock()
mock_response.trailing_metadata.return_value = (('result', 'ok'), )
mock_response.__iter__.return_value = iter([
untrusted_runner_pb2.FileChunk(data='A'),
untrusted_runner_pb2.FileChunk(data='B'),
untrusted_runner_pb2.FileChunk(data='C'),
])
self.mock.stub().CopyFileFrom.return_value = mock_response
self.assertTrue(file_host.copy_file_from_worker('/file', '/file'))
with open('/file') as f:
self.assertEqual(f.read(), 'ABC')
def _process_corpus_crashes(context, result):
"""Process crashes found in the corpus."""
# Default Testcase entity values.
crash_revision = result.revision
job_type = environment.get_value("JOB_NAME")
minimized_arguments = "%TESTCASE% " + context.fuzz_target.binary
project_name = data_handler.get_project_name(job_type)
comment = "Fuzzer %s generated corpus testcase crashed (r%s)" % (
context.fuzz_target.project_qualified_name(),
crash_revision,
)
# Generate crash reports.
for crash in result.crashes:
existing_testcase = data_handler.find_testcase(project_name,
crash.crash_type,
crash.crash_state,
crash.security_flag)
if existing_testcase:
continue
# Upload/store testcase.
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
unit_path = os.path.join(context.bad_units_path,
os.path.basename(crash.unit_path))
# Prevent the worker from escaping out of |context.bad_units_path|.
if not file_host.is_directory_parent(unit_path,
context.bad_units_path):
raise CorpusPruningException("Invalid units path from worker.")
file_host.copy_file_from_worker(crash.unit_path, unit_path)
else:
unit_path = crash.unit_path
with open(unit_path, "rb") as f:
key = blobs.write_blob(f)
# Set the absolute_path property of the Testcase to a file in FUZZ_INPUTS
# instead of the local quarantine directory.
absolute_testcase_path = os.path.join(
environment.get_value("FUZZ_INPUTS"), "testcase")
testcase_id = data_handler.store_testcase(
crash=crash,
fuzzed_keys=key,
minimized_keys="",
regression="",
fixed="",
one_time_crasher_flag=False,
crash_revision=crash_revision,
comment=comment,
absolute_path=absolute_testcase_path,
fuzzer_name=context.fuzz_target.engine,
fully_qualified_fuzzer_name=context.fuzz_target.
fully_qualified_name(),
job_type=job_type,
archived=False,
archive_filename="",
binary_flag=True,
http_flag=False,
gestures=None,
redzone=DEFAULT_REDZONE,
disable_ubsan=False,
minidump_keys=None,
window_argument=None,
timeout_multiplier=1.0,
minimized_arguments=minimized_arguments,
)
# Set fuzzer_binary_name in testcase metadata.
testcase = data_handler.get_testcase_by_id(testcase_id)
testcase.set_metadata("fuzzer_binary_name", result.fuzzer_binary_name)
issue_metadata = engine_common.get_all_issue_metadata_for_testcase(
testcase)
if issue_metadata:
for key, value in issue_metadata.items():
testcase.set_metadata(key, value, update_testcase=False)
testcase.put()
# Create additional tasks for testcase (starting with minimization).
testcase = data_handler.get_testcase_by_id(testcase_id)
task_creation.create_tasks(testcase)
def test_copy_file_from_worker_does_not_exist(self): """Tests remote copy_file_from_worker (does not exist).""" src_path = os.path.join(self.tmp_dir, 'DOES_NOT_EXIST') dest_path = os.path.join(self.tmp_dir, 'dst') self.assertFalse(file_host.copy_file_from_worker(src_path, dest_path)) self.assertFalse(os.path.exists(dest_path))
def _run_libfuzzer_tool(tool_name,
testcase,
testcase_file_path,
timeout,
expected_crash_state,
set_dedup_flags=False):
"""Run libFuzzer tool to either minimize or cleanse."""
memory_tool_options_var = environment.get_current_memory_tool_var()
saved_memory_tool_options = environment.get_value(memory_tool_options_var)
def _set_dedup_flags():
"""Allow libFuzzer to do its own crash comparison during minimization."""
memory_tool_options = environment.get_memory_tool_options(
memory_tool_options_var)
memory_tool_options['symbolize'] = 1
memory_tool_options['dedup_token_length'] = 3
environment.set_memory_tool_options(memory_tool_options_var,
memory_tool_options)
def _unset_dedup_flags():
"""Reset memory tool options."""
# This is needed so that when we re-run, we can symbolize ourselves
# (ignoring inline frames).
environment.set_value(memory_tool_options_var, saved_memory_tool_options)
output_file_path = get_temporary_file_name(testcase_file_path)
rebased_output_file_path = output_file_path
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
file_host.copy_file_to_worker(
testcase_file_path, file_host.rebase_to_worker_root(testcase_file_path))
rebased_output_file_path = file_host.rebase_to_worker_root(output_file_path)
arguments = environment.get_value('APP_ARGS', '')
arguments += (' --cf-{tool_name}-timeout={timeout} '
'--cf-{tool_name}-to={output_file_path}').format(
tool_name=tool_name,
output_file_path=rebased_output_file_path,
timeout=timeout)
command = tests.get_command_line_for_application(
file_to_run=testcase_file_path,
app_args=arguments,
needs_http=testcase.http_flag)
logs.log('Executing command: %s' % command)
if set_dedup_flags:
_set_dedup_flags()
# A small buffer is added to the timeout to allow the current test to
# finish, and file to be written. Since we should terminate beforehand, a
# long delay only slows fuzzing in cases where it's necessary.
_, _, output = process_handler.run_process(command, timeout=timeout + 60)
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
file_host.copy_file_from_worker(rebased_output_file_path, output_file_path)
if set_dedup_flags:
_unset_dedup_flags()
if not os.path.exists(output_file_path):
logs.log_warn('LibFuzzer %s run failed.' % tool_name, output=output)
return None, None
# Ensure that the crash parameters match. It's possible that we will
# minimize/cleanse to an unrelated bug, such as a timeout.
crash_result = _run_libfuzzer_testcase(testcase, output_file_path)
state = crash_result.get_symbolized_data()
security_flag = crash_result.is_security_issue()
if (security_flag != testcase.security_flag or
state.crash_state != expected_crash_state):
logs.log_warn('Ignoring unrelated crash.\n'
'State: %s (expected %s)\n'
'Security: %s (expected %s)\n'
'Output: %s\n' %
(state.crash_state, expected_crash_state, security_flag,
testcase.security_flag, state.crash_stacktrace))
return None, None
with open(output_file_path, 'rb') as file_handle:
minimized_keys = blobs.write_blob(file_handle)
testcase.minimized_keys = minimized_keys
testcase.put()
return output_file_path, crash_result
