def get_runner(fuzzer_path, temp_dir=None):
"""Get a libfuzzer runner."""
use_minijail = environment.get_value('USE_MINIJAIL')
build_dir = environment.get_value('BUILD_DIR')
dataflow_build_dir = environment.get_value('DATAFLOW_BUILD_DIR')
if use_minijail:
# Set up chroot and runner.
if environment.is_chromeos_system_job():
minijail_chroot = minijail.ChromeOSChroot(build_dir)
else:
minijail_chroot = minijail.MinijailChroot(base_dir=temp_dir)
# While it's possible for dynamic binaries to run without this, they need
# to be accessible for symbolization etc. For simplicity we bind BUILD_DIR
# to the same location within the chroot, which leaks the directory
# structure of CF but this shouldn't be a big deal.
minijail_chroot.add_binding(
minijail.ChrootBinding(build_dir, build_dir, False))
if dataflow_build_dir:
minijail_chroot.add_binding(
minijail.ChrootBinding(dataflow_build_dir, dataflow_build_dir,
False))
# Also bind the build dir to /out to make it easier to hardcode references
# to data files.
minijail_chroot.add_binding(
minijail.ChrootBinding(build_dir, '/out', False))
minijail_bin = os.path.join(minijail_chroot.directory, 'bin')
shell.create_directory(minijail_bin)
# Set up /bin with llvm-symbolizer to allow symbolized stacktraces.
# Don't copy if it already exists (e.g. ChromeOS chroot jail).
llvm_symbolizer_source_path = environment.get_llvm_symbolizer_path()
llvm_symbolizer_destination_path = os.path.join(
minijail_bin, 'llvm-symbolizer')
if not os.path.exists(llvm_symbolizer_destination_path):
shutil.copy(llvm_symbolizer_source_path,
llvm_symbolizer_destination_path)
# copy /bin/sh, necessary for system().
if not environment.is_chromeos_system_job():
# The chroot has its own shell we don't need to copy (and probably
# shouldn't because of library differences).
shutil.copy(os.path.realpath('/bin/sh'),
os.path.join(minijail_bin, 'sh'))
runner = MinijailLibFuzzerRunner(fuzzer_path, minijail_chroot)
elif environment.platform() == 'FUCHSIA':
runner = FuchsiaQemuLibFuzzerRunner(fuzzer_path)
else:
runner = LibFuzzerRunner(fuzzer_path)
return runner
def main():
"""
Converts all input files from [input-dir] into numpy, runs
fuzzer binary on them to get coverage, and saves inputs and
labels as numpy arrays in a ready-to-use format by `ProgramDataset`
in data_utils.py.
Outputs two directories and two files.
Directories include
> label dir (data/[dataset_name]/labels/label-{num}.npy)
> input dir (data/[dataset_name]/inputs/input-{num}.npy)
Files include
> input filenames list file (data/[dataset_name]/input_file_names.json)
> branch numbers file (data/[dataset_name]/branches.json)
"""
args = parse_args()
output_dir = os.path.join(constants.DATASET_DIR, args.dataset_name)
# Argument error checking.
if os.path.isdir(output_dir):
print('Error: {} is already a named dataset directory (check under {}/).'
.format(output_dir, constants.DATASET_DIR))
sys.exit()
cutoff_args = [
args.cutoff_std, args.cutoff_percentile, args.median_mult_cutoff
]
num_cutoff_args = sum(cutoff_arg is not None for cutoff_arg in cutoff_args)
if num_cutoff_args > 1:
print('Error: Only one of [--cutoff-std, --cutoff-percentile, ' +
'--median-mult-cutoff] may be specified.')
sys.exit()
if num_cutoff_args == 0:
print('Warning: Proceeding with no dataset pruning.')
print('All inputs will be kept, regardless of length.\n')
os.makedirs(os.path.join(output_dir, constants.STANDARD_INPUT_DIR))
os.makedirs(os.path.join(output_dir, constants.STANDARD_LABEL_DIR))
# For fuzz target coverage printing.
os.environ['ASAN_SYMBOLIZER_PATH'] = environment.get_llvm_symbolizer_path()
process_all(
args.input_dir,
output_dir,
args.fuzz_target_binary,
args.cutoff_std,
args.cutoff_percentile,
args.median_mult_cutoff,
pad=args.pad)
def symbolize_stacktrace(unsymbolized_crash_stacktrace,
enable_inline_frames=True):
"""Symbolize a crash stacktrace."""
if environment.is_trusted_host():
from bot.untrusted_runner import symbolize_host
return symbolize_host.symbolize_stacktrace(unsymbolized_crash_stacktrace,
enable_inline_frames)
platform = environment.platform()
if platform == 'WINDOWS':
# Windows Clang ASAN provides symbolized stacktraces anyway.
return unsymbolized_crash_stacktrace
if platform == 'FUCHSIA':
# Fuchsia Clang ASAN provides symbolized stacktraces anyway.
return unsymbolized_crash_stacktrace
# FIXME: Support symbolization on ChromeOS device.
if platform == 'CHROMEOS':
return unsymbolized_crash_stacktrace
# Initialize variables.
global llvm_symbolizer_path
global pipes
global stack_inlining
global symbolizers
pipes = []
stack_inlining = str(enable_inline_frames).lower()
symbolizers = {}
# Make sure we have a llvm symbolizer for this platform.
llvm_symbolizer_path = environment.get_llvm_symbolizer_path()
if not llvm_symbolizer_path:
return unsymbolized_crash_stacktrace
# Disable buffering for stdout.
disable_buffering()
loop = SymbolizationLoop(
binary_path_filter=filter_binary_path,
dsym_hint_producer=chrome_dsym_hints)
symbolized_crash_stacktrace = loop.process_stacktrace(
unsymbolized_crash_stacktrace)
return symbolized_crash_stacktrace
def terminate_stale_application_instances():
"""Kill stale instances of the application running for this command."""
if environment.is_trusted_host():
from bot.untrusted_runner import remote_process_host
remote_process_host.terminate_stale_application_instances()
return
# Stale instance cleanup is sometimes disabled for local testing.
if not environment.get_value('KILL_STALE_INSTANCES', True):
return
additional_process_to_kill = environment.get_value(
'ADDITIONAL_PROCESSES_TO_KILL')
builds_directory = environment.get_value('BUILDS_DIR')
llvm_symbolizer_filename = environment.get_executable_filename(
'llvm-symbolizer')
platform = environment.platform()
start_time = time.time()
processes_to_kill = []
# Avoid killing the test binary when running the reproduce tool. It is
# commonly in-use on the side on developer workstations.
if not environment.get_value('REPRODUCE_TOOL'):
app_name = environment.get_value('APP_NAME')
processes_to_kill += [app_name]
if additional_process_to_kill:
processes_to_kill += additional_process_to_kill.split(' ')
processes_to_kill = [x for x in processes_to_kill if x]
if platform == 'ANDROID':
# Cleanup any stale adb connections.
device_serial = environment.get_value('ANDROID_SERIAL')
adb_search_string = 'adb -s %s' % device_serial
# Terminate llvm symbolizer processes matching exact path. This is important
# for Android where multiple device instances run on same host.
llvm_symbolizer_path = environment.get_llvm_symbolizer_path()
terminate_processes_matching_cmd_line(
[adb_search_string, llvm_symbolizer_path], kill=True)
# Make sure device is online and rooted.
android.adb.run_as_root()
# Make sure to reset SE Linux Permissive Mode (might be lost in reboot).
android.settings.change_se_linux_to_permissive_mode()
# Make sure that device forwarder is running (might be lost in reboot or
# process crash).
android.device.setup_host_and_device_forwarder_if_needed()
# Make sure that package optimization is complete (might be triggered due to
# unexpected circumstances).
android.app.wait_until_optimization_complete()
# Reset application state, which kills its pending instances and re-grants
# the storage permissions.
android.app.reset()
elif platform == 'WINDOWS':
processes_to_kill += [
'cdb.exe',
'handle.exe',
'msdt.exe',
'openwith.exe',
'WerFault.exe',
llvm_symbolizer_filename,
]
terminate_processes_matching_names(processes_to_kill, kill=True)
terminate_processes_matching_cmd_line(builds_directory, kill=True)
# Artifical sleep to let the processes get terminated.
time.sleep(1)
else:
# Handle Linux and Mac platforms.
processes_to_kill += [
'addr2line',
'atos',
'chrome-devel-sandbox',
'gdb',
'nacl_helper',
'xdotool',
llvm_symbolizer_filename,
]
terminate_processes_matching_names(processes_to_kill, kill=True)
terminate_processes_matching_cmd_line(builds_directory, kill=True)
duration = int(time.time() - start_time)
if duration >= 5:
logs.log('Process kill took longer than usual - %s.' %
str(datetime.timedelta(seconds=duration)))