def get_corpus_directory(input_directory, project_qualified_name): """Get the corpus directory given a project qualified fuzz target name.""" corpus_directory = os.path.join(input_directory, project_qualified_name) if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import file_host corpus_directory = file_host.rebase_to_worker_root(corpus_directory) # Create corpus directory if it does not exist already. if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import file_host file_host.create_directory(corpus_directory, create_intermediates=True) else: shell.create_directory(corpus_directory) return corpus_directory
def run(): """Run update task.""" # Since this code is particularly sensitive for bot stability, continue # execution but store the exception if anything goes wrong during one of these # steps. try: # Update heartbeat with current time. data_handler.update_heartbeat() # Check overall free disk space. If we are running too low, clear all # data directories like builds, fuzzers, data bundles, etc. shell.clear_data_directories_on_low_disk_space() # Download new layout tests once per day. update_tests_if_needed() except Exception: logs.log_error('Error occurred while running update task.') # Even if there is an exception in one of the other steps, we want to try to # update the source. If for some reason the source code update fails, it is # not necessary to run the init scripts. try: # If there is a newer revision, exit and let run.py update the source code. if get_newer_source_revision() is not None: if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import host host.update_worker() sys.exit(0) # Run platform specific initialization scripts. run_platform_init_scripts() except Exception: logs.log_error('Error occurred while running update task.')
def clear_build_urls_directory(): """Clears the build url directory.""" remove_directory(environment.get_value('BUILD_URLS_DIR'), recreate=True) if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import file_host file_host.clear_build_urls_directory()
def update_environment_for_job(environment_string): """Process the environment variable string included with a job.""" # Now parse the job's environment definition. environment_values = ( environment.parse_environment_definition(environment_string)) for key, value in six.iteritems(environment_values): environment.set_value(key, value) # If we share the build with another job type, force us to be a custom binary # job type. if environment.get_value('SHARE_BUILD_WITH_JOB_TYPE'): environment.set_value('CUSTOM_BINARY', True) # Allow the default FUZZ_TEST_TIMEOUT and MAX_TESTCASES to be overridden on # machines that are preempted more often. fuzz_test_timeout_override = environment.get_value( 'FUZZ_TEST_TIMEOUT_OVERRIDE') if fuzz_test_timeout_override: environment.set_value('FUZZ_TEST_TIMEOUT', fuzz_test_timeout_override) max_testcases_override = environment.get_value('MAX_TESTCASES_OVERRIDE') if max_testcases_override: environment.set_value('MAX_TESTCASES', max_testcases_override) if environment.is_trusted_host(): environment_values['JOB_NAME'] = environment.get_value('JOB_NAME') from clusterfuzz._internal.bot.untrusted_runner import \ environment as worker_environment worker_environment.update_environment(environment_values)
def legalize_filenames(file_paths): """Convert the name of every file in |file_paths| a name that is legal on Windows. Returns list of legally named files.""" # TODO(metzman): Support legalizing filenames when called on trusted host, but # file paths exist on untrusted workers. This is fine for now since Linux is # the only supported platform on OSS-Fuzz and this functionality is not needed # in OSS-Fuzz. if environment.is_trusted_host(): return file_paths illegal_chars = {'<', '>', ':', '\\', '|', '?', '*'} failed_to_move_files = [] legally_named = [] for file_path in file_paths: file_dir_path, basename = os.path.split(file_path) if not any(char in illegal_chars for char in basename): legally_named.append(file_path) continue # Hash file to get new name since it also lets us get rid of duplicates, # will not cause collisions for different files and makes things more # consistent (since libFuzzer uses hashes). sha1sum = utils.file_hash(file_path) new_file_path = os.path.join(file_dir_path, sha1sum) try: shutil.move(file_path, new_file_path) legally_named.append(new_file_path) except OSError: failed_to_move_files.append((file_path, new_file_path)) if failed_to_move_files: logs.log_error( 'Failed to rename files.', failed_to_move_files=failed_to_move_files) return legally_named
def _low_disk_space_threshold(): """Get the low disk space threshold.""" if environment.is_trusted_host(ensure_connected=False): # Trusted hosts can run with less free space as they do not store builds or # corpora. return _TRUSTED_HOST_LOW_DISK_SPACE_THRESHOLD return _DEFAULT_LOW_DISK_SPACE_THRESHOLD
def close_queue(queue_to_close): """Close the queue.""" if environment.is_trusted_host(): # We don't use multiprocessing.Queue on trusted hosts. return try: queue_to_close.close() except: logs.log_error('Unable to close queue.')
def get_process(): """Return a multiprocessing process object (with bug fixes).""" if environment.is_trusted_host(): # forking/multiprocessing is unsupported because of the RPC connection. return threading.Thread # FIXME(unassigned): Remove this hack after real bug is fixed. # pylint: disable=protected-access multiprocessing.current_process()._identity = () return multiprocessing.Process
def clear_testcase_directories(): """Clears the testcase directories.""" remove_directory(environment.get_value('FUZZ_INPUTS'), recreate=True) remove_directory(environment.get_value('FUZZ_INPUTS_DISK'), recreate=True) if environment.is_android() and environment.get_value('ANDROID_SERIAL'): from clusterfuzz._internal.platforms import android android.device.clear_testcase_directory() if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import file_host file_host.clear_testcase_directories()
def get_queue(): """Return a multiprocessing queue object.""" if environment.is_trusted_host(): # We don't use multiprocessing.Process on trusted hosts. No need to use # multiprocessing.Queue. return queue.Queue() try: result_queue = multiprocessing.Queue() except: # FIXME: Invalid cross-device link error. Happens sometimes with # chroot jobs even though /dev/shm and /run/shm are mounted. logs.log_error('Unable to get multiprocessing queue.') return None return result_queue
def get_issue_metadata(fuzz_target_path, extension): """Get issue metadata.""" metadata_file_path = fuzzer_utils.get_supporting_file( fuzz_target_path, extension) if environment.is_trusted_host(): metadata_file_path = fuzzer_utils.get_file_from_untrusted_worker( metadata_file_path) if not os.path.exists(metadata_file_path): return [] with open(metadata_file_path) as handle: return utils.parse_delimited(handle, delimiter='\n', strip=True, remove_empty=True)
def get_all_issue_metadata_for_testcase(testcase): """Get issue related metadata given a testcase.""" if environment.is_trusted_host(): # Not applicable. return None fuzz_target = testcase.get_fuzz_target() if not fuzz_target: return None build_dir = environment.get_value('BUILD_DIR') target_path = find_fuzzer_path(build_dir, fuzz_target.binary) if not target_path: logs.log_error('Failed to find target path for ' + fuzz_target.binary) return None return get_all_issue_metadata(target_path)
def symbolize_stacktrace(unsymbolized_crash_stacktrace, enable_inline_frames=True): """Symbolize a crash stacktrace.""" if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import symbolize_host return symbolize_host.symbolize_stacktrace(unsymbolized_crash_stacktrace, enable_inline_frames) platform = environment.platform() if platform == 'WINDOWS': # Windows Clang ASAN provides symbolized stacktraces anyway. return unsymbolized_crash_stacktrace if platform == 'FUCHSIA': # Fuchsia Clang ASAN provides symbolized stacktraces anyway. return unsymbolized_crash_stacktrace # FIXME: Support symbolization on ChromeOS device. if platform == 'CHROMEOS': return unsymbolized_crash_stacktrace # Initialize variables. global llvm_symbolizer_path global pipes global stack_inlining global symbolizers pipes = [] stack_inlining = str(enable_inline_frames).lower() symbolizers = {} # Make sure we have a llvm symbolizer for this platform. llvm_symbolizer_path = environment.get_llvm_symbolizer_path() if not llvm_symbolizer_path: return unsymbolized_crash_stacktrace # Disable buffering for stdout. disable_buffering() loop = SymbolizationLoop( binary_path_filter=filter_binary_path, dsym_hint_producer=chrome_dsym_hints) symbolized_crash_stacktrace = loop.process_stacktrace( unsymbolized_crash_stacktrace) return symbolized_crash_stacktrace
def get_additional_issue_metadata(fuzz_target_path): """Return the additional metadata fields given a fuzz target path. The data will be a JSON-formatted dictionary.""" metadata_file_path = fuzzer_utils.get_supporting_file( fuzz_target_path, METADATA_FILE_EXTENSION) if environment.is_trusted_host(): metadata_file_path = fuzzer_utils.get_file_from_untrusted_worker( metadata_file_path) if not os.path.exists(metadata_file_path): return {} with open(metadata_file_path) as handle: try: return json.load(handle) except (ValueError, TypeError): logs.log_error('Invalid metadata file format.', path=metadata_file_path) return {}
def get_fuzz_target_options(fuzz_target_path): """Return a FuzzerOptions for the given target, or None if it does not exist.""" options_file_path = fuzzer_utils.get_supporting_file( fuzz_target_path, OPTIONS_FILE_EXTENSION) if environment.is_trusted_host(): options_file_path = fuzzer_utils.get_file_from_untrusted_worker( options_file_path) if not os.path.exists(options_file_path): return None options_cwd = os.path.dirname(options_file_path) try: return FuzzerOptions(options_file_path, cwd=options_cwd) except FuzzerOptionsException: logs.log_error('Invalid options file: %s.' % options_file_path) return None
def get_issue_owners(fuzz_target_path): """Return list of owner emails given a fuzz target path. Format of an owners file is described at: https://cs.chromium.org/chromium/src/third_party/depot_tools/owners.py """ owners_file_path = fuzzer_utils.get_supporting_file( fuzz_target_path, OWNERS_FILE_EXTENSION) if environment.is_trusted_host(): owners_file_path = fuzzer_utils.get_file_from_untrusted_worker( owners_file_path) if not os.path.exists(owners_file_path): return [] owners = [] with open(owners_file_path, 'r') as owners_file_handle: owners_file_content = owners_file_handle.read() for line in owners_file_content.splitlines(): stripped_line = line.strip() if not stripped_line: # Ignore empty lines. continue if stripped_line.startswith('#'): # Ignore comment lines. continue if stripped_line == '*': # Not of any use, we can't add everyone as owner with this. continue if (stripped_line.startswith('per-file') or stripped_line.startswith('file:')): # Don't have a source checkout, so ignore. continue if '@' not in stripped_line: # Bad email address. continue owners.append(stripped_line) return owners
def clear_temp_directory(clear_user_profile_directories=True): """Clear the temporary directories.""" temp_directory = environment.get_value('BOT_TMPDIR') remove_directory(temp_directory, recreate=True) test_temp_directory = environment.get_value('TEST_TMPDIR') if test_temp_directory != temp_directory: remove_directory(test_temp_directory, recreate=True) if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import file_host file_host.clear_temp_directory() if not clear_user_profile_directories: return user_profile_root_directory = environment.get_value( 'USER_PROFILE_ROOT_DIR') if not user_profile_root_directory: return remove_directory(user_profile_root_directory, recreate=True)
def engine_reproduce(engine_impl, target_name, testcase_path, arguments, timeout): """Do engine reproduction.""" if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import tasks_host return tasks_host.engine_reproduce(engine_impl, target_name, testcase_path, arguments, timeout) build_dir = environment.get_value('BUILD_DIR') target_path = engine_common.find_fuzzer_path(build_dir, target_name) if not target_path: raise TargetNotFoundError('Failed to find target ' + target_name) result = engine_impl.reproduce(target_path, testcase_path, list(arguments), timeout) # This matches the check in process_handler.run_process. if not result.return_code and \ (crash_analyzer.is_memory_tool_crash(result.output) or crash_analyzer.is_check_failure_crash(result.output)): result.return_code = 1 return result
def terminate_hung_threads(threads): """Terminate hung threads.""" start_time = time.time() while time.time() - start_time < THREAD_FINISH_WAIT_TIME: if not any([thread.is_alive() for thread in threads]): # No threads are alive, so we're done. return time.sleep(0.1) logs.log_warn('Hang detected.', snapshot=get_runtime_snapshot()) if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import host # Bail out on trusted hosts since we're using threads and can't clean up. host.host_exit_no_return() # Terminate all threads that are still alive. try: [thread.terminate() for thread in threads if thread.is_alive()] except: pass
def _is_data_bundle_up_to_date(data_bundle, data_bundle_directory): """Return true if the data bundle is up to date, false otherwise.""" sync_file_path = _get_data_bundle_sync_file_path(data_bundle_directory) if environment.is_trusted_host() and data_bundle.sync_to_worker: from clusterfuzz._internal.bot.untrusted_runner import file_host worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path) shell.remove_file(sync_file_path) file_host.copy_file_from_worker(worker_sync_file_path, sync_file_path) if not os.path.exists(sync_file_path): return False last_sync_time = datetime.datetime.utcfromtimestamp( utils.read_data_from_file(sync_file_path)) # Check if we recently synced. if not dates.time_has_expired( last_sync_time, seconds=_DATA_BUNDLE_SYNC_INTERVAL_IN_SECONDS): return True # For search index data bundle, we don't sync them from bucket. Instead, we # rely on the fuzzer to generate testcases periodically. if _is_search_index_data_bundle(data_bundle.name): return False # Check when the bucket url had last updates. If no new updates, no need to # update directory. bucket_url = data_handler.get_data_bundle_bucket_url(data_bundle.name) last_updated_time = storage.last_updated(bucket_url) if last_updated_time and last_sync_time > last_updated_time: logs.log('Data bundle %s has no new content from last sync.' % data_bundle.name) return True return False
def _do_run_testcase_and_return_result_in_queue(crash_queue, thread_index, file_path, gestures, env_copy, upload_output=False): """Run a single testcase and return crash results in the crash queue.""" try: # Run testcase and check whether a crash occurred or not. return_code, crash_time, output = run_testcase(thread_index, file_path, gestures, env_copy) # Pull testcase directory to host to get any stats files. if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import file_host file_host.pull_testcases_from_worker() # Analyze the crash. crash_output = _get_crash_output(output) crash_result = CrashResult(return_code, crash_time, crash_output) # To provide consistency between stats and logs, we use timestamp taken # from stats when uploading logs and testcase. if upload_output: log_time = _get_testcase_time(file_path) if crash_result.is_crash(): # Initialize resource list with the testcase path. resource_list = [file_path] resource_list += get_resource_paths(crash_output) # Store the crash stack file in the crash stacktrace directory # with filename as the hash of the testcase path. crash_stacks_directory = environment.get_value( 'CRASH_STACKTRACES_DIR') stack_file_path = os.path.join(crash_stacks_directory, utils.string_hash(file_path)) utils.write_data_to_file(crash_output, stack_file_path) # Put crash/no-crash results in the crash queue. crash_queue.put( Crash(file_path=file_path, crash_time=crash_time, return_code=return_code, resource_list=resource_list, gestures=gestures, stack_file_path=stack_file_path)) # Don't upload uninteresting testcases (no crash) or if there is no log to # correlate it with (not upload_output). if upload_output: upload_testcase(file_path, log_time) if upload_output: # Include full output for uploaded logs (crash output, merge output, etc). crash_result_full = CrashResult(return_code, crash_time, output) log = prepare_log_for_upload(crash_result_full.get_stacktrace(), return_code) upload_log(log, log_time) except Exception: logs.log_error('Exception occurred while running ' 'run_testcase_and_return_result_in_queue.')
def _process_corpus_crashes(context, result): """Process crashes found in the corpus.""" # Default Testcase entity values. crash_revision = result.revision job_type = environment.get_value('JOB_NAME') minimized_arguments = '%TESTCASE% ' + context.fuzz_target.binary project_name = data_handler.get_project_name(job_type) comment = 'Fuzzer %s generated corpus testcase crashed (r%s)' % ( context.fuzz_target.project_qualified_name(), crash_revision) # Generate crash reports. for crash in result.crashes: existing_testcase = data_handler.find_testcase(project_name, crash.crash_type, crash.crash_state, crash.security_flag) if existing_testcase: continue # Upload/store testcase. if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import file_host unit_path = os.path.join(context.bad_units_path, os.path.basename(crash.unit_path)) # Prevent the worker from escaping out of |context.bad_units_path|. if not file_host.is_directory_parent(unit_path, context.bad_units_path): raise CorpusPruningException('Invalid units path from worker.') file_host.copy_file_from_worker(crash.unit_path, unit_path) else: unit_path = crash.unit_path with open(unit_path, 'rb') as f: key = blobs.write_blob(f) # Set the absolute_path property of the Testcase to a file in FUZZ_INPUTS # instead of the local quarantine directory. absolute_testcase_path = os.path.join( environment.get_value('FUZZ_INPUTS'), 'testcase') testcase_id = data_handler.store_testcase( crash=crash, fuzzed_keys=key, minimized_keys='', regression='', fixed='', one_time_crasher_flag=False, crash_revision=crash_revision, comment=comment, absolute_path=absolute_testcase_path, fuzzer_name=context.fuzz_target.engine, fully_qualified_fuzzer_name=context.fuzz_target. fully_qualified_name(), job_type=job_type, archived=False, archive_filename='', binary_flag=True, http_flag=False, gestures=None, redzone=DEFAULT_REDZONE, disable_ubsan=False, minidump_keys=None, window_argument=None, timeout_multiplier=1.0, minimized_arguments=minimized_arguments) # Set fuzzer_binary_name in testcase metadata. testcase = data_handler.get_testcase_by_id(testcase_id) testcase.set_metadata('fuzzer_binary_name', result.fuzzer_binary_name) issue_metadata = engine_common.get_all_issue_metadata_for_testcase( testcase) if issue_metadata: for key, value in issue_metadata.items(): testcase.set_metadata(key, value, update_testcase=False) testcase.put() # Create additional tasks for testcase (starting with minimization). testcase = data_handler.get_testcase_by_id(testcase_id) task_creation.create_tasks(testcase)
def main(): """Prepare the configuration options and start requesting tasks.""" logs.configure('run_bot') root_directory = environment.get_value('ROOT_DIR') if not root_directory: print('Please set ROOT_DIR environment variable to the root of the source ' 'checkout before running. Exiting.') print('For an example, check init.bash in the local directory.') return dates.initialize_timezone_from_environment() environment.set_bot_environment() monitor.initialize() if not profiler.start_if_needed('python_profiler_bot'): sys.exit(-1) fuzzers_init.run() if environment.is_trusted_host(ensure_connected=False): from clusterfuzz._internal.bot.untrusted_runner import host host.init() if environment.is_untrusted_worker(): # Track revision since we won't go into the task_loop. update_task.track_revision() from clusterfuzz._internal.bot.untrusted_runner import \ untrusted as untrusted_worker untrusted_worker.start_server() assert False, 'Unreachable code' while True: # task_loop should be an infinite loop, # unless we run into an exception. error_stacktrace, clean_exit, task_payload = task_loop() # Print the error trace to the console. if not clean_exit: print('Exception occurred while running "%s".' % task_payload) print('-' * 80) print(error_stacktrace) print('-' * 80) should_terminate = ( clean_exit or errors.error_in_list(error_stacktrace, errors.BOT_ERROR_TERMINATION_LIST)) if should_terminate: return logs.log_error( 'Task exited with exception (payload="%s").' % task_payload, error_stacktrace=error_stacktrace) should_hang = errors.error_in_list(error_stacktrace, errors.BOT_ERROR_HANG_LIST) if should_hang: logs.log('Start hanging forever.') while True: # Sleep to avoid consuming 100% of CPU. time.sleep(60) # See if our run timed out, if yes bail out. if data_handler.bot_run_timed_out(): return
def run_process(cmdline, current_working_directory=None, timeout=DEFAULT_TEST_TIMEOUT, need_shell=False, gestures=None, env_copy=None, testcase_run=True, ignore_children=True): """Executes a process with a given command line and other parameters.""" if environment.is_trusted_host() and testcase_run: from clusterfuzz._internal.bot.untrusted_runner import remote_process_host return remote_process_host.run_process(cmdline, current_working_directory, timeout, need_shell, gestures, env_copy, testcase_run, ignore_children) if gestures is None: gestures = [] if env_copy: os.environ.update(env_copy) # FIXME(mbarbella): Using LAUNCHER_PATH here is error prone. It forces us to # do certain operations before fuzzer setup (e.g. bad build check). launcher = environment.get_value('LAUNCHER_PATH') # This is used when running scripts on native linux OS and not on the device. # E.g. running a fuzzer to generate testcases or launcher script. plt = environment.platform() runs_on_device = environment.is_android(plt) or plt == 'FUCHSIA' if runs_on_device and (not testcase_run or launcher): plt = 'LINUX' is_android = environment.is_android(plt) # Lower down testcase timeout slightly to account for time for crash analysis. timeout -= CRASH_ANALYSIS_TIME # LeakSanitizer hack - give time for stdout/stderr processing. lsan = environment.get_value('LSAN', False) if lsan: timeout -= LSAN_ANALYSIS_TIME # Initialize variables. adb_output = None process_output = '' process_status = None return_code = 0 process_poll_interval = environment.get_value('PROCESS_POLL_INTERVAL', 0.5) start_time = time.time() watch_for_process_exit = (environment.get_value('WATCH_FOR_PROCESS_EXIT') if is_android else True) window_list = [] # Get gesture start time from last element in gesture list. gestures = copy.deepcopy(gestures) if gestures and gestures[-1].startswith('Trigger'): gesture_start_time = int(gestures[-1].split(':')[1]) gestures.pop() else: gesture_start_time = timeout // 2 if is_android: # Clear the log upfront. android.logger.clear_log() # Run the app. adb_output = android.adb.run_command(cmdline, timeout=timeout) else: cmd = shell.get_command(cmdline) process_output = mozprocess.processhandler.StoreOutput() process_status = ProcessStatus() try: process_handle = mozprocess.ProcessHandlerMixin( cmd, args=None, cwd=current_working_directory, shell=need_shell, processOutputLine=[process_output], onFinish=[process_status], ignore_children=ignore_children) start_process(process_handle) except: logs.log_error('Exception occurred when running command: %s.' % cmdline) return None, None, '' while True: time.sleep(process_poll_interval) # Run the gestures at gesture_start_time or in case we didn't find windows # in the last try. if (gestures and time.time() - start_time >= gesture_start_time and not window_list): # In case, we don't find any windows, we increment the gesture start time # so that the next check is after 1 second. gesture_start_time += 1 if plt == 'LINUX': linux.gestures.run_gestures(gestures, process_handle.pid, process_status, start_time, timeout, window_list) elif plt == 'WINDOWS': windows.gestures.run_gestures(gestures, process_handle.pid, process_status, start_time, timeout, window_list) elif is_android: android.gestures.run_gestures(gestures, start_time, timeout) # TODO(mbarbella): We add a fake window here to prevent gestures on # Android from getting executed more than once. window_list = ['FAKE'] if time.time() - start_time >= timeout: break # Collect the process output. output = (android.logger.log_output() if is_android else b'\n'.join(process_output.output)) output = utils.decode_to_unicode(output) if crash_analyzer.is_memory_tool_crash(output): break # Check if we need to bail out on process exit. if watch_for_process_exit: # If |watch_for_process_exit| is set, then we already completed running # our app launch command. So, we can bail out. if is_android: break # On desktop, we bail out as soon as the process finishes. if process_status and process_status.finished: # Wait for process shutdown and set return code. process_handle.wait(timeout=PROCESS_CLEANUP_WAIT_TIME) break # Process output based on platform. if is_android: # Get current log output. If device is in reboot mode, logcat automatically # waits for device to be online. time.sleep(ANDROID_CRASH_LOGCAT_WAIT_TIME) output = android.logger.log_output() if android.constants.LOW_MEMORY_REGEX.search(output): # If the device is low on memory, we should force reboot and bail out to # prevent device from getting in a frozen state. logs.log('Device is low on memory, rebooting.', output=output) android.adb.hard_reset() android.adb.wait_for_device() elif android.adb.time_since_last_reboot() < time.time() - start_time: # Check if a reboot has happened, if yes, append log output before reboot # and kernel logs content to output. log_before_last_reboot = android.logger.log_output_before_last_reboot( ) kernel_log = android.adb.get_kernel_log_content() output = '%s%s%s%s%s' % ( log_before_last_reboot, utils.get_line_seperator('Device rebooted'), output, utils.get_line_seperator('Kernel Log'), kernel_log) # Make sure to reset SE Linux Permissive Mode. This can be done cheaply # in ~0.15 sec and is needed especially between runs for kernel crashes. android.adb.run_as_root() android.settings.change_se_linux_to_permissive_mode() return_code = 1 # Add output from adb to the front. if adb_output: output = '%s\n\n%s' % (adb_output, output) # Kill the application if it is still running. We do this at the end to # prevent this from adding noise to the logcat output. task_name = environment.get_value('TASK_NAME') child_process_termination_pattern = environment.get_value( 'CHILD_PROCESS_TERMINATION_PATTERN') if task_name == 'fuzz' and child_process_termination_pattern: # In some cases, we do not want to terminate the application after each # run to avoid long startup times (e.g. for chrome). Terminate processes # matching a particular pattern for light cleanup in this case. android.adb.kill_processes_and_children_matching_name( child_process_termination_pattern) else: # There is no special termination behavior. Simply stop the application. android.app.stop() else: # Get the return code in case the process has finished already. # If the process hasn't finished, return_code will be None which is what # callers expect unless the output indicates a crash. return_code = process_handle.poll() # If the process is still running, then terminate it. if not process_status.finished: launcher_with_interpreter = shell.get_execute_command( launcher) if launcher else None if (launcher_with_interpreter and cmdline.startswith(launcher_with_interpreter)): # If this was a launcher script, we KILL all child processes created # except for APP_NAME. # It is expected that, if the launcher script terminated normally, it # cleans up all the child processes it created itself. terminate_root_and_child_processes(process_handle.pid) else: try: # kill() here actually sends SIGTERM on posix. process_handle.kill() except: pass if lsan: time.sleep(LSAN_ANALYSIS_TIME) output = b'\n'.join(process_output.output) output = utils.decode_to_unicode(output) # X Server hack when max client reached. if ('Maximum number of clients reached' in output or 'Unable to get connection to X server' in output): logs.log_error('Unable to connect to X server, exiting.') os.system('sudo killall -9 Xvfb blackbox >/dev/null 2>&1') sys.exit(0) if testcase_run and (crash_analyzer.is_memory_tool_crash(output) or crash_analyzer.is_check_failure_crash(output)): return_code = 1 # If a crash is found, then we add the memory state as well. if return_code and is_android: ps_output = android.adb.get_ps_output() if ps_output: output += utils.get_line_seperator('Memory Statistics') output += ps_output if return_code: logs.log_warn('Process (%s) ended with exit code (%s).' % (repr(cmdline), str(return_code)), output=output) return return_code, round(time.time() - start_time, 1), output
def get_command_line_for_application(file_to_run='', user_profile_index=0, app_path=None, app_args=None, needs_http=False, write_command_line_file=False, get_arguments_only=False): """Returns the complete command line required to execute application.""" if app_args is None: app_args = environment.get_value('APP_ARGS') if app_path is None: app_path = environment.get_value('APP_PATH') if not app_path: # No APP_PATH is available for e.g. grey box fuzzers. return '' additional_command_line_flags = get_additional_command_line_flags( file_to_run) app_args_append_testcase = environment.get_value( 'APP_ARGS_APPEND_TESTCASE') app_directory = environment.get_value('APP_DIR') app_name = environment.get_value('APP_NAME') apps_argument = environment.get_value('APPS_ARG') crash_stacks_directory = environment.get_value('CRASH_STACKTRACES_DIR') debugger = environment.get_value('DEBUGGER_PATH') device_testcases_directory = android.constants.DEVICE_TESTCASES_DIR fuzzer_directory = environment.get_value('FUZZER_DIR') extension_argument = environment.get_value('EXTENSION_ARG') input_directory = environment.get_value('INPUT_DIR') launcher = environment.get_value('LAUNCHER_PATH') is_android = environment.is_android() root_directory = environment.get_value('ROOT_DIR') temp_directory = environment.get_value('BOT_TMPDIR') user_profile_argument = environment.get_value('USER_PROFILE_ARG') window_argument = environment.get_value('WINDOW_ARG') user_profile_directory = get_user_profile_directory(user_profile_index) # Create user profile directory and setup contents if needed. setup_user_profile_directory_if_needed(user_profile_directory) # Handle spaces in APP_PATH. # If application path has spaces, then we need to quote it. if ' ' in app_path: app_path = '"%s"' % app_path interpreter = shell.get_interpreter(app_name) if get_arguments_only: # If we are only returning the arguments, do not return the application # path or anything else required to run it such as an interpreter. app_path = '' elif interpreter: # Prepend command with interpreter if it is a script. app_path = '%s %s' % (interpreter, app_path) # Start creating the command line. command = '' # Rebase the file_to_run and launcher paths to the worker's root. if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import file_host file_to_run = file_host.rebase_to_worker_root(file_to_run) launcher = file_host.rebase_to_worker_root(launcher) # Default case. testcase_path = file_to_run testcase_filename = os.path.basename(testcase_path) testcase_directory = os.path.dirname(testcase_path) testcase_file_url = utils.file_path_to_file_url(testcase_path) testcase_http_url = '' # Determine where |testcase_file_url| should point depending on platform and # whether or not a launcher script is used. if file_to_run: if launcher: # In the case of launcher scripts, the testcase file to be run resides on # the host running the launcher script. Thus |testcase_file_url|, which # may point to a location on the device for Android job types, does not # apply. Instead, the launcher script should be passed the original file # to run. By setting |testcase_file_url| to |file_to_run|, we avoid # duplicating job definitions solely for supporting launcher scripts. testcase_file_url = file_to_run # Jobs that have a launcher script which needs to be run on the host will # have app_name == launcher. In this case don't prepend launcher to # command - just use app_name. if os.path.basename(launcher) != app_name: launcher_with_interpreter = shell.get_execute_command(launcher) command += launcher_with_interpreter + ' ' elif is_android: # Android-specific testcase path fixup for fuzzers that don't rely on # launcher scripts. local_testcases_directory = environment.get_value('FUZZ_INPUTS') # Check if the file to run is in fuzzed testcases folder. If yes, then we # can substitute with a local device path. Otherwise, it is part of some # data bundle with resource dependencies and we just need to use http # host forwarder for that. if file_to_run.startswith(local_testcases_directory): testcase_relative_path = ( file_to_run[len(local_testcases_directory) + 1:]) testcase_path = os.path.join(device_testcases_directory, testcase_relative_path) testcase_file_url = utils.file_path_to_file_url(testcase_path) else: # Force use of host_forwarder based on comment above. needs_http = True # Check if the testcase needs to be loaded over http. # TODO(ochang): Make this work for trusted/untrusted. http_ip = '127.0.0.1' http_port_1 = environment.get_value('HTTP_PORT_1', 8000) relative_testcase_path = file_to_run[len(input_directory + os.path.sep):] relative_testcase_path = relative_testcase_path.replace('\\', '/') testcase_http_url = 'http://%s:%d/%s' % (http_ip, http_port_1, relative_testcase_path) if needs_http: # TODO(unassigned): Support https. testcase_file_url = testcase_http_url testcase_path = testcase_http_url # Compose app arguments. all_app_args = '' if user_profile_argument: all_app_args += ' %s=%s' % (user_profile_argument, user_profile_directory) if extension_argument and EXTENSIONS_PREFIX in testcase_filename: all_app_args += ' %s=%s' % (extension_argument, testcase_directory) if apps_argument and APPS_PREFIX in testcase_filename: all_app_args += ' %s=%s' % (apps_argument, testcase_directory) if window_argument: all_app_args += ' %s' % window_argument if additional_command_line_flags: all_app_args += ' %s' % additional_command_line_flags.strip() if app_args: all_app_args += ' %s' % app_args.strip() # Append %TESTCASE% at end if no testcase pattern is found in app arguments. if not utils.sub_string_exists_in( ['%TESTCASE%', '%TESTCASE_FILE_URL%', '%TESTCASE_HTTP_URL%'], all_app_args) and app_args_append_testcase: all_app_args += ' %TESTCASE%' all_app_args = all_app_args.strip() # Build the actual command to run now. if debugger: command += '%s ' % debugger if app_path: command += app_path if all_app_args: command += ' %s' % all_app_args command = command.replace('%APP_DIR%', app_directory) command = command.replace('%CRASH_STACKTRACES_DIR%', crash_stacks_directory) command = command.replace('%DEVICE_TESTCASES_DIR%', device_testcases_directory) command = command.replace('%FUZZER_DIR%', fuzzer_directory) command = command.replace('%INPUT_DIR%', input_directory) command = command.replace('%ROOT_DIR%', root_directory) command = command.replace('%TESTCASE%', testcase_path) command = command.replace('%TESTCASE_FILE_URL%', testcase_file_url) command = command.replace('%TESTCASE_HTTP_URL%', testcase_http_url) command = command.replace('%TMP_DIR%', temp_directory) command = command.replace('%USER_PROFILE_DIR%', user_profile_directory) if is_android and not launcher: # Initial setup phase for command line. if write_command_line_file: android.adb.write_command_line_file(command, app_path) return android.app.get_launch_command(all_app_args, testcase_path, testcase_file_url) # Decide which directory we will run the application from. # We are using |app_directory| since it helps to locate pdbs # in same directory, other dependencies, etc. if os.path.exists(app_directory): os.chdir(app_directory) return str(command)
def setup_testcase(testcase, job_type, fuzzer_override=None): """Sets up the testcase and needed dependencies like fuzzer, data bundle, etc.""" fuzzer_name = fuzzer_override or testcase.fuzzer_name task_name = environment.get_value('TASK_NAME') testcase_fail_wait = environment.get_value('FAIL_WAIT') testcase_id = testcase.key.id() # Clear testcase directories. shell.clear_testcase_directories() # Adjust the test timeout value if this is coming from an user uploaded # testcase. if testcase.uploader_email: _set_timeout_value_from_user_upload(testcase_id) # Update the fuzzer if necessary in order to get the updated data bundle. if fuzzer_name: try: update_successful = update_fuzzer_and_data_bundles(fuzzer_name) except errors.InvalidFuzzerError: # Close testcase and don't recreate tasks if this fuzzer is invalid. testcase.open = False testcase.fixed = 'NA' testcase.set_metadata('fuzzer_was_deleted', True) logs.log_error('Closed testcase %d with invalid fuzzer %s.' % (testcase_id, fuzzer_name)) error_message = 'Fuzzer %s no longer exists' % fuzzer_name data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) return None, None, None if not update_successful: error_message = 'Unable to setup fuzzer %s' % fuzzer_name data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) tasks.add_task(task_name, testcase_id, job_type, wait_time=testcase_fail_wait) return None, None, None # Extract the testcase and any of its resources to the input directory. file_list, input_directory, testcase_file_path = unpack_testcase(testcase) if not file_list: error_message = 'Unable to setup testcase %s' % testcase_file_path data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) tasks.add_task(task_name, testcase_id, job_type, wait_time=testcase_fail_wait) return None, None, None # For Android/Fuchsia, we need to sync our local testcases directory with the # one on the device. if environment.is_android(): _copy_testcase_to_device_and_setup_environment(testcase, testcase_file_path) # Push testcases to worker. if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import file_host file_host.push_testcases_to_worker() # Copy global blacklist into local blacklist. is_lsan_enabled = environment.get_value('LSAN') if is_lsan_enabled: # Get local blacklist without this testcase's entry. leak_blacklist.copy_global_to_local_blacklist( excluded_testcase=testcase) prepare_environment_for_testcase(testcase, job_type, task_name) return file_list, input_directory, testcase_file_path
def update_fuzzer_and_data_bundles(fuzzer_name): """Update the fuzzer with a given name if necessary.""" fuzzer = data_types.Fuzzer.query( data_types.Fuzzer.name == fuzzer_name).get() if not fuzzer: logs.log_error('No fuzzer exists with name %s.' % fuzzer_name) raise errors.InvalidFuzzerError # Set some helper environment variables. fuzzer_directory = get_fuzzer_directory(fuzzer_name) environment.set_value('FUZZER_DIR', fuzzer_directory) environment.set_value('UNTRUSTED_CONTENT', fuzzer.untrusted_content) # If the fuzzer generates large testcases or a large number of small ones # that don't fit on tmpfs, then use the larger disk directory. if fuzzer.has_large_testcases: testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK') environment.set_value('FUZZ_INPUTS', testcase_disk_directory) # Adjust the test timeout, if user has provided one. if fuzzer.timeout: environment.set_value('TEST_TIMEOUT', fuzzer.timeout) # Increase fuzz test timeout if the fuzzer timeout is higher than its # current value. fuzz_test_timeout = environment.get_value('FUZZ_TEST_TIMEOUT') if fuzz_test_timeout and fuzz_test_timeout < fuzzer.timeout: environment.set_value('FUZZ_TEST_TIMEOUT', fuzzer.timeout) # Adjust the max testcases if this fuzzer has specified a lower limit. max_testcases = environment.get_value('MAX_TESTCASES') if fuzzer.max_testcases and fuzzer.max_testcases < max_testcases: environment.set_value('MAX_TESTCASES', fuzzer.max_testcases) # Check for updates to this fuzzer. version_file = os.path.join(fuzzer_directory, '.%s_version' % fuzzer_name) if (not fuzzer.builtin and revisions.needs_update(version_file, fuzzer.revision)): logs.log('Fuzzer update was found, updating.') # Clear the old fuzzer directory if it exists. if not shell.remove_directory(fuzzer_directory, recreate=True): logs.log_error('Failed to clear fuzzer directory.') return None # Copy the archive to local disk and unpack it. archive_path = os.path.join(fuzzer_directory, fuzzer.filename) if not blobs.read_blob_to_disk(fuzzer.blobstore_key, archive_path): logs.log_error('Failed to copy fuzzer archive.') return None try: archive.unpack(archive_path, fuzzer_directory) except Exception: error_message = ( 'Failed to unpack fuzzer archive %s ' '(bad archive or unsupported format).') % fuzzer.filename logs.log_error(error_message) fuzzer_logs.upload_script_log('Fatal error: ' + error_message, fuzzer_name=fuzzer_name) return None fuzzer_path = os.path.join(fuzzer_directory, fuzzer.executable_path) if not os.path.exists(fuzzer_path): error_message = ( 'Fuzzer executable %s not found. ' 'Check fuzzer configuration.') % fuzzer.executable_path logs.log_error(error_message) fuzzer_logs.upload_script_log('Fatal error: ' + error_message, fuzzer_name=fuzzer_name) return None # Make fuzzer executable. os.chmod(fuzzer_path, 0o750) # Cleanup unneeded archive. shell.remove_file(archive_path) # Save the current revision of this fuzzer in a file for later checks. revisions.write_revision_to_revision_file(version_file, fuzzer.revision) logs.log('Updated fuzzer to revision %d.' % fuzzer.revision) _clear_old_data_bundles_if_needed() # Setup data bundles associated with this fuzzer. data_bundles = ndb_utils.get_all_from_query( data_types.DataBundle.query( data_types.DataBundle.name == fuzzer.data_bundle_name)) for data_bundle in data_bundles: if not update_data_bundle(fuzzer, data_bundle): return None # Setup environment variable for launcher script path. if fuzzer.launcher_script: fuzzer_launcher_path = os.path.join(fuzzer_directory, fuzzer.launcher_script) environment.set_value('LAUNCHER_PATH', fuzzer_launcher_path) # For launcher script usecase, we need the entire fuzzer directory on the # worker. if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import file_host worker_fuzzer_directory = file_host.rebase_to_worker_root( fuzzer_directory) file_host.copy_directory_to_worker(fuzzer_directory, worker_fuzzer_directory, replace=True) return fuzzer
def update_data_bundle(fuzzer, data_bundle): """Updates a data bundle to the latest version.""" # This module can't be in the global imports due to appengine issues # with multiprocessing and psutil imports. from clusterfuzz._internal.google_cloud_utils import gsutil # If we are using a data bundle on NFS, it is expected that our testcases # will usually be large enough that we would fill up our tmpfs directory # pretty quickly. So, change it to use an on-disk directory. if not data_bundle.is_local: testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK') environment.set_value('FUZZ_INPUTS', testcase_disk_directory) data_bundle_directory = get_data_bundle_directory(fuzzer.name) if not data_bundle_directory: logs.log_error('Failed to setup data bundle %s.' % data_bundle.name) return False if not shell.create_directory(data_bundle_directory, create_intermediates=True): logs.log_error('Failed to create data bundle %s directory.' % data_bundle.name) return False # Check if data bundle is up to date. If yes, skip the update. if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory): logs.log('Data bundle was recently synced, skip.') return True # Fetch lock for this data bundle. if not _fetch_lock_for_data_bundle_update(data_bundle): logs.log_error('Failed to lock data bundle %s.' % data_bundle.name) return False # Re-check if another bot did the sync already. If yes, skip. if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory): logs.log('Another bot finished the sync, skip.') _release_lock_for_data_bundle_update(data_bundle) return True time_before_sync_start = time.time() # No need to sync anything if this is a search index data bundle. In that # case, the fuzzer will generate testcases from a gcs bucket periodically. if not _is_search_index_data_bundle(data_bundle.name): bucket_url = data_handler.get_data_bundle_bucket_url(data_bundle.name) if environment.is_trusted_host() and data_bundle.sync_to_worker: from clusterfuzz._internal.bot.untrusted_runner import corpus_manager from clusterfuzz._internal.bot.untrusted_runner import file_host worker_data_bundle_directory = file_host.rebase_to_worker_root( data_bundle_directory) file_host.create_directory(worker_data_bundle_directory, create_intermediates=True) result = corpus_manager.RemoteGSUtilRunner().rsync( bucket_url, worker_data_bundle_directory, delete=False) else: result = gsutil.GSUtilRunner().rsync(bucket_url, data_bundle_directory, delete=False) if result.return_code != 0: logs.log_error('Failed to sync data bundle %s: %s.' % (data_bundle.name, result.output)) _release_lock_for_data_bundle_update(data_bundle) return False # Update the testcase list file. testcase_manager.create_testcase_list_file(data_bundle_directory) # Write last synced time in the sync file. sync_file_path = _get_data_bundle_sync_file_path(data_bundle_directory) utils.write_data_to_file(time_before_sync_start, sync_file_path) if environment.is_trusted_host() and data_bundle.sync_to_worker: from clusterfuzz._internal.bot.untrusted_runner import file_host worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path) file_host.copy_file_to_worker(sync_file_path, worker_sync_file_path) # Release acquired lock. _release_lock_for_data_bundle_update(data_bundle) return True
def get_fuzz_targets(path): """Get list of fuzz targets paths.""" if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import file_host return file_host.get_fuzz_targets(path) return get_fuzz_targets_local(path)
def terminate_stale_application_instances(): """Kill stale instances of the application running for this command.""" if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import remote_process_host remote_process_host.terminate_stale_application_instances() return # Stale instance cleanup is sometimes disabled for local testing. if not environment.get_value('KILL_STALE_INSTANCES', True): return additional_process_to_kill = environment.get_value( 'ADDITIONAL_PROCESSES_TO_KILL') builds_directory = environment.get_value('BUILDS_DIR') llvm_symbolizer_filename = environment.get_executable_filename( 'llvm-symbolizer') platform = environment.platform() start_time = time.time() processes_to_kill = [] # Avoid killing the test binary when running the reproduce tool. It is # commonly in-use on the side on developer workstations. if not environment.get_value('REPRODUCE_TOOL'): app_name = environment.get_value('APP_NAME') processes_to_kill += [app_name] if additional_process_to_kill: processes_to_kill += additional_process_to_kill.split(' ') processes_to_kill = [x for x in processes_to_kill if x] if environment.is_android(platform): # Cleanup any stale adb connections. device_serial = environment.get_value('ANDROID_SERIAL') adb_search_string = 'adb -s %s' % device_serial # Terminate llvm symbolizer processes matching exact path. This is important # for Android where multiple device instances run on same host. llvm_symbolizer_path = environment.get_llvm_symbolizer_path() terminate_processes_matching_cmd_line( [adb_search_string, llvm_symbolizer_path], kill=True) # Make sure device is online and rooted. android.adb.run_as_root() # Make sure to reset SE Linux Permissive Mode (might be lost in reboot). android.settings.change_se_linux_to_permissive_mode() # Make sure that device forwarder is running (might be lost in reboot or # process crash). android.device.setup_host_and_device_forwarder_if_needed() # Make sure that package optimization is complete (might be triggered due to # unexpected circumstances). android.app.wait_until_optimization_complete() # Reset application state, which kills its pending instances and re-grants # the storage permissions. android.app.reset() elif platform == 'WINDOWS': processes_to_kill += [ 'cdb.exe', 'handle.exe', 'msdt.exe', 'openwith.exe', 'WerFault.exe', llvm_symbolizer_filename, ] terminate_processes_matching_names(processes_to_kill, kill=True) terminate_processes_matching_cmd_line(builds_directory, kill=True) # Artifical sleep to let the processes get terminated. time.sleep(1) elif platform == 'FUCHSIA': processes_to_kill += [ 'undercoat', llvm_symbolizer_filename, ] terminate_processes_matching_names(processes_to_kill, kill=True) else: # Handle Linux and Mac platforms. processes_to_kill += [ 'addr2line', 'atos', 'chrome-devel-sandbox', 'gdb', 'nacl_helper', 'xdotool', llvm_symbolizer_filename, ] terminate_processes_matching_names(processes_to_kill, kill=True) terminate_processes_matching_cmd_line(builds_directory, kill=True) duration = int(time.time() - start_time) if duration >= 5: logs.log('Process kill took longer than usual - %s.' % str(datetime.timedelta(seconds=duration)))