def process_testcase(engine_name, tool_name, target_name, arguments, testcase_path, output_path, timeout): """Process testcase on untrusted worker.""" if tool_name == 'minimize': operation = untrusted_runner_pb2.ProcessTestcaseRequest.MINIMIZE else: operation = untrusted_runner_pb2.ProcessTestcaseRequest.CLEANSE rebased_testcase_path = file_host.rebase_to_worker_root(testcase_path) file_host.copy_file_to_worker(testcase_path, rebased_testcase_path) request = untrusted_runner_pb2.ProcessTestcaseRequest( engine=engine_name, operation=operation, target_name=target_name, arguments=arguments, testcase_path=file_host.rebase_to_worker_root(testcase_path), output_path=file_host.rebase_to_worker_root(output_path), timeout=timeout) response = host.stub().ProcessTestcase(request) rebased_output_path = file_host.rebase_to_worker_root(output_path) file_host.copy_file_from_worker(rebased_output_path, output_path) return engine.ReproduceResult(list(response.command), response.return_code, response.time_executed, response.output)
def get_file_from_untrusted_worker(worker_file_path): """Gets file from an untrusted worker to local. Local file stays in the temp folder until the end of task or can be explicitly deleted by the caller.""" from bot.untrusted_runner import file_host with tempfile.NamedTemporaryFile(delete=False, dir=get_temp_dir()) as f: local_file_path = f.name file_host.copy_file_from_worker(worker_file_path, local_file_path) return local_file_path
def test_copy_file_from_worker_failure(self): """Test file_host.copy_file_from_worker (failure).""" mock_response = mock.MagicMock() mock_response.trailing_metadata.return_value = (('result', 'invalid-path'),) self.mock.stub().CopyFileFrom.return_value = mock_response self.assertFalse(file_host.copy_file_from_worker('/file', '/file')) self.assertFalse(os.path.exists('/file'))
def test_copy_file_from_worker(self): """Tests remote copy_file_from_worker.""" src_path = os.path.join(self.tmp_dir, 'src') with open(src_path, 'w') as f: f.write(TEST_FILE_CONTENTS) dest_path = os.path.join(self.tmp_dir, 'dst') self.assertTrue(file_host.copy_file_from_worker(src_path, dest_path)) with open(dest_path) as f: self.assertEqual(f.read(), TEST_FILE_CONTENTS)
def _is_data_bundle_up_to_date(data_bundle, data_bundle_directory): """Return true if the data bundle is up to date, false otherwise.""" sync_file_path = _get_data_bundle_sync_file_path(data_bundle_directory) if environment.is_trusted_host() and data_bundle.sync_to_worker: from bot.untrusted_runner import file_host worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path) shell.remove_file(sync_file_path) file_host.copy_file_from_worker(worker_sync_file_path, sync_file_path) if not os.path.exists(sync_file_path): return False last_sync_time = datetime.datetime.utcfromtimestamp( utils.read_data_from_file(sync_file_path)) # Check if we recently synced. if not dates.time_has_expired( last_sync_time, seconds=_DATA_BUNDLE_SYNC_INTERVAL_IN_SECONDS): return True # For search index data bundle, we don't sync them from bucket. Instead, we # rely on the fuzzer to generate testcases periodically. if _is_search_index_data_bundle(data_bundle.name): return False # Check when the bucket url had last updates. If no new updates, no need to # update directory. bucket_url = data_handler.get_data_bundle_bucket_url(data_bundle.name) last_updated_time = storage.last_updated(bucket_url) if last_updated_time and last_sync_time > last_updated_time: logs.log("Data bundle %s has no new content from last sync." % data_bundle.name) return True return False
def test_copy_file_from_worker(self): """Test file_host.copy_file_from_worker.""" mock_response = mock.MagicMock() mock_response.trailing_metadata.return_value = (('result', 'ok'), ) mock_response.__iter__.return_value = iter([ untrusted_runner_pb2.FileChunk(data='A'), untrusted_runner_pb2.FileChunk(data='B'), untrusted_runner_pb2.FileChunk(data='C'), ]) self.mock.stub().CopyFileFrom.return_value = mock_response self.assertTrue(file_host.copy_file_from_worker('/file', '/file')) with open('/file') as f: self.assertEqual(f.read(), 'ABC')
def _process_corpus_crashes(context, result): """Process crashes found in the corpus.""" # Default Testcase entity values. crash_revision = result.revision job_type = environment.get_value("JOB_NAME") minimized_arguments = "%TESTCASE% " + context.fuzz_target.binary project_name = data_handler.get_project_name(job_type) comment = "Fuzzer %s generated corpus testcase crashed (r%s)" % ( context.fuzz_target.project_qualified_name(), crash_revision, ) # Generate crash reports. for crash in result.crashes: existing_testcase = data_handler.find_testcase(project_name, crash.crash_type, crash.crash_state, crash.security_flag) if existing_testcase: continue # Upload/store testcase. if environment.is_trusted_host(): from bot.untrusted_runner import file_host unit_path = os.path.join(context.bad_units_path, os.path.basename(crash.unit_path)) # Prevent the worker from escaping out of |context.bad_units_path|. if not file_host.is_directory_parent(unit_path, context.bad_units_path): raise CorpusPruningException("Invalid units path from worker.") file_host.copy_file_from_worker(crash.unit_path, unit_path) else: unit_path = crash.unit_path with open(unit_path, "rb") as f: key = blobs.write_blob(f) # Set the absolute_path property of the Testcase to a file in FUZZ_INPUTS # instead of the local quarantine directory. absolute_testcase_path = os.path.join( environment.get_value("FUZZ_INPUTS"), "testcase") testcase_id = data_handler.store_testcase( crash=crash, fuzzed_keys=key, minimized_keys="", regression="", fixed="", one_time_crasher_flag=False, crash_revision=crash_revision, comment=comment, absolute_path=absolute_testcase_path, fuzzer_name=context.fuzz_target.engine, fully_qualified_fuzzer_name=context.fuzz_target. fully_qualified_name(), job_type=job_type, archived=False, archive_filename="", binary_flag=True, http_flag=False, gestures=None, redzone=DEFAULT_REDZONE, disable_ubsan=False, minidump_keys=None, window_argument=None, timeout_multiplier=1.0, minimized_arguments=minimized_arguments, ) # Set fuzzer_binary_name in testcase metadata. testcase = data_handler.get_testcase_by_id(testcase_id) testcase.set_metadata("fuzzer_binary_name", result.fuzzer_binary_name) issue_metadata = engine_common.get_all_issue_metadata_for_testcase( testcase) if issue_metadata: for key, value in issue_metadata.items(): testcase.set_metadata(key, value, update_testcase=False) testcase.put() # Create additional tasks for testcase (starting with minimization). testcase = data_handler.get_testcase_by_id(testcase_id) task_creation.create_tasks(testcase)
def test_copy_file_from_worker_does_not_exist(self): """Tests remote copy_file_from_worker (does not exist).""" src_path = os.path.join(self.tmp_dir, 'DOES_NOT_EXIST') dest_path = os.path.join(self.tmp_dir, 'dst') self.assertFalse(file_host.copy_file_from_worker(src_path, dest_path)) self.assertFalse(os.path.exists(dest_path))
def _run_libfuzzer_tool(tool_name, testcase, testcase_file_path, timeout, expected_crash_state, set_dedup_flags=False): """Run libFuzzer tool to either minimize or cleanse.""" memory_tool_options_var = environment.get_current_memory_tool_var() saved_memory_tool_options = environment.get_value(memory_tool_options_var) def _set_dedup_flags(): """Allow libFuzzer to do its own crash comparison during minimization.""" memory_tool_options = environment.get_memory_tool_options( memory_tool_options_var) memory_tool_options['symbolize'] = 1 memory_tool_options['dedup_token_length'] = 3 environment.set_memory_tool_options(memory_tool_options_var, memory_tool_options) def _unset_dedup_flags(): """Reset memory tool options.""" # This is needed so that when we re-run, we can symbolize ourselves # (ignoring inline frames). environment.set_value(memory_tool_options_var, saved_memory_tool_options) output_file_path = get_temporary_file_name(testcase_file_path) rebased_output_file_path = output_file_path if environment.is_trusted_host(): from bot.untrusted_runner import file_host file_host.copy_file_to_worker( testcase_file_path, file_host.rebase_to_worker_root(testcase_file_path)) rebased_output_file_path = file_host.rebase_to_worker_root(output_file_path) arguments = environment.get_value('APP_ARGS', '') arguments += (' --cf-{tool_name}-timeout={timeout} ' '--cf-{tool_name}-to={output_file_path}').format( tool_name=tool_name, output_file_path=rebased_output_file_path, timeout=timeout) command = tests.get_command_line_for_application( file_to_run=testcase_file_path, app_args=arguments, needs_http=testcase.http_flag) logs.log('Executing command: %s' % command) if set_dedup_flags: _set_dedup_flags() # A small buffer is added to the timeout to allow the current test to # finish, and file to be written. Since we should terminate beforehand, a # long delay only slows fuzzing in cases where it's necessary. _, _, output = process_handler.run_process(command, timeout=timeout + 60) if environment.is_trusted_host(): from bot.untrusted_runner import file_host file_host.copy_file_from_worker(rebased_output_file_path, output_file_path) if set_dedup_flags: _unset_dedup_flags() if not os.path.exists(output_file_path): logs.log_warn('LibFuzzer %s run failed.' % tool_name, output=output) return None, None # Ensure that the crash parameters match. It's possible that we will # minimize/cleanse to an unrelated bug, such as a timeout. crash_result = _run_libfuzzer_testcase(testcase, output_file_path) state = crash_result.get_symbolized_data() security_flag = crash_result.is_security_issue() if (security_flag != testcase.security_flag or state.crash_state != expected_crash_state): logs.log_warn('Ignoring unrelated crash.\n' 'State: %s (expected %s)\n' 'Security: %s (expected %s)\n' 'Output: %s\n' % (state.crash_state, expected_crash_state, security_flag, testcase.security_flag, state.crash_stacktrace)) return None, None with open(output_file_path, 'rb') as file_handle: minimized_keys = blobs.write_blob(file_handle) testcase.minimized_keys = minimized_keys testcase.put() return output_file_path, crash_result