def get_runner(fuzzer_path, temp_dir=None): """Get a libfuzzer runner.""" use_minijail = environment.get_value('USE_MINIJAIL') build_dir = environment.get_value('BUILD_DIR') dataflow_build_dir = environment.get_value('DATAFLOW_BUILD_DIR') if use_minijail: # Set up chroot and runner. if environment.is_chromeos_system_job(): minijail_chroot = minijail.ChromeOSChroot(build_dir) else: minijail_chroot = minijail.MinijailChroot(base_dir=temp_dir) # While it's possible for dynamic binaries to run without this, they need # to be accessible for symbolization etc. For simplicity we bind BUILD_DIR # to the same location within the chroot, which leaks the directory # structure of CF but this shouldn't be a big deal. minijail_chroot.add_binding( minijail.ChrootBinding(build_dir, build_dir, False)) if dataflow_build_dir: minijail_chroot.add_binding( minijail.ChrootBinding(dataflow_build_dir, dataflow_build_dir, False)) # Also bind the build dir to /out to make it easier to hardcode references # to data files. minijail_chroot.add_binding( minijail.ChrootBinding(build_dir, '/out', False)) minijail_bin = os.path.join(minijail_chroot.directory, 'bin') shell.create_directory(minijail_bin) # Set up /bin with llvm-symbolizer to allow symbolized stacktraces. # Don't copy if it already exists (e.g. ChromeOS chroot jail). llvm_symbolizer_source_path = environment.get_llvm_symbolizer_path() llvm_symbolizer_destination_path = os.path.join( minijail_bin, 'llvm-symbolizer') if not os.path.exists(llvm_symbolizer_destination_path): shutil.copy(llvm_symbolizer_source_path, llvm_symbolizer_destination_path) # copy /bin/sh, necessary for system(). if not environment.is_chromeos_system_job(): # The chroot has its own shell we don't need to copy (and probably # shouldn't because of library differences). shutil.copy(os.path.realpath('/bin/sh'), os.path.join(minijail_bin, 'sh')) runner = MinijailLibFuzzerRunner(fuzzer_path, minijail_chroot) elif environment.platform() == 'FUCHSIA': runner = FuchsiaQemuLibFuzzerRunner(fuzzer_path) else: runner = LibFuzzerRunner(fuzzer_path) return runner
def main(): """ Converts all input files from [input-dir] into numpy, runs fuzzer binary on them to get coverage, and saves inputs and labels as numpy arrays in a ready-to-use format by `ProgramDataset` in data_utils.py. Outputs two directories and two files. Directories include > label dir (data/[dataset_name]/labels/label-{num}.npy) > input dir (data/[dataset_name]/inputs/input-{num}.npy) Files include > input filenames list file (data/[dataset_name]/input_file_names.json) > branch numbers file (data/[dataset_name]/branches.json) """ args = parse_args() output_dir = os.path.join(constants.DATASET_DIR, args.dataset_name) # Argument error checking. if os.path.isdir(output_dir): print('Error: {} is already a named dataset directory (check under {}/).' .format(output_dir, constants.DATASET_DIR)) sys.exit() cutoff_args = [ args.cutoff_std, args.cutoff_percentile, args.median_mult_cutoff ] num_cutoff_args = sum(cutoff_arg is not None for cutoff_arg in cutoff_args) if num_cutoff_args > 1: print('Error: Only one of [--cutoff-std, --cutoff-percentile, ' + '--median-mult-cutoff] may be specified.') sys.exit() if num_cutoff_args == 0: print('Warning: Proceeding with no dataset pruning.') print('All inputs will be kept, regardless of length.\n') os.makedirs(os.path.join(output_dir, constants.STANDARD_INPUT_DIR)) os.makedirs(os.path.join(output_dir, constants.STANDARD_LABEL_DIR)) # For fuzz target coverage printing. os.environ['ASAN_SYMBOLIZER_PATH'] = environment.get_llvm_symbolizer_path() process_all( args.input_dir, output_dir, args.fuzz_target_binary, args.cutoff_std, args.cutoff_percentile, args.median_mult_cutoff, pad=args.pad)
def symbolize_stacktrace(unsymbolized_crash_stacktrace, enable_inline_frames=True): """Symbolize a crash stacktrace.""" if environment.is_trusted_host(): from bot.untrusted_runner import symbolize_host return symbolize_host.symbolize_stacktrace(unsymbolized_crash_stacktrace, enable_inline_frames) platform = environment.platform() if platform == 'WINDOWS': # Windows Clang ASAN provides symbolized stacktraces anyway. return unsymbolized_crash_stacktrace if platform == 'FUCHSIA': # Fuchsia Clang ASAN provides symbolized stacktraces anyway. return unsymbolized_crash_stacktrace # FIXME: Support symbolization on ChromeOS device. if platform == 'CHROMEOS': return unsymbolized_crash_stacktrace # Initialize variables. global llvm_symbolizer_path global pipes global stack_inlining global symbolizers pipes = [] stack_inlining = str(enable_inline_frames).lower() symbolizers = {} # Make sure we have a llvm symbolizer for this platform. llvm_symbolizer_path = environment.get_llvm_symbolizer_path() if not llvm_symbolizer_path: return unsymbolized_crash_stacktrace # Disable buffering for stdout. disable_buffering() loop = SymbolizationLoop( binary_path_filter=filter_binary_path, dsym_hint_producer=chrome_dsym_hints) symbolized_crash_stacktrace = loop.process_stacktrace( unsymbolized_crash_stacktrace) return symbolized_crash_stacktrace
def terminate_stale_application_instances(): """Kill stale instances of the application running for this command.""" if environment.is_trusted_host(): from bot.untrusted_runner import remote_process_host remote_process_host.terminate_stale_application_instances() return # Stale instance cleanup is sometimes disabled for local testing. if not environment.get_value('KILL_STALE_INSTANCES', True): return additional_process_to_kill = environment.get_value( 'ADDITIONAL_PROCESSES_TO_KILL') builds_directory = environment.get_value('BUILDS_DIR') llvm_symbolizer_filename = environment.get_executable_filename( 'llvm-symbolizer') platform = environment.platform() start_time = time.time() processes_to_kill = [] # Avoid killing the test binary when running the reproduce tool. It is # commonly in-use on the side on developer workstations. if not environment.get_value('REPRODUCE_TOOL'): app_name = environment.get_value('APP_NAME') processes_to_kill += [app_name] if additional_process_to_kill: processes_to_kill += additional_process_to_kill.split(' ') processes_to_kill = [x for x in processes_to_kill if x] if platform == 'ANDROID': # Cleanup any stale adb connections. device_serial = environment.get_value('ANDROID_SERIAL') adb_search_string = 'adb -s %s' % device_serial # Terminate llvm symbolizer processes matching exact path. This is important # for Android where multiple device instances run on same host. llvm_symbolizer_path = environment.get_llvm_symbolizer_path() terminate_processes_matching_cmd_line( [adb_search_string, llvm_symbolizer_path], kill=True) # Make sure device is online and rooted. android.adb.run_as_root() # Make sure to reset SE Linux Permissive Mode (might be lost in reboot). android.settings.change_se_linux_to_permissive_mode() # Make sure that device forwarder is running (might be lost in reboot or # process crash). android.device.setup_host_and_device_forwarder_if_needed() # Make sure that package optimization is complete (might be triggered due to # unexpected circumstances). android.app.wait_until_optimization_complete() # Reset application state, which kills its pending instances and re-grants # the storage permissions. android.app.reset() elif platform == 'WINDOWS': processes_to_kill += [ 'cdb.exe', 'handle.exe', 'msdt.exe', 'openwith.exe', 'WerFault.exe', llvm_symbolizer_filename, ] terminate_processes_matching_names(processes_to_kill, kill=True) terminate_processes_matching_cmd_line(builds_directory, kill=True) # Artifical sleep to let the processes get terminated. time.sleep(1) else: # Handle Linux and Mac platforms. processes_to_kill += [ 'addr2line', 'atos', 'chrome-devel-sandbox', 'gdb', 'nacl_helper', 'xdotool', llvm_symbolizer_filename, ] terminate_processes_matching_names(processes_to_kill, kill=True) terminate_processes_matching_cmd_line(builds_directory, kill=True) duration = int(time.time() - start_time) if duration >= 5: logs.log('Process kill took longer than usual - %s.' % str(datetime.timedelta(seconds=duration)))