def is_memory_tool_crash(stacktrace):
"""Return true if it is a memory debugging tool crash."""
# Job-specific generic checks.
crash_signature = environment.get_value('CRASH_SIGNATURE')
if crash_signature and re.search(crash_signature, stacktrace):
return True
# Android specific check.
# FIXME: Share this regex with stack_analyzer.
if (environment.is_android() and re.match(
r'.*signal.*\(SIG.*fault addr ([^ ]*)', stacktrace, re.DOTALL)):
return True
# Check if we have a complete stacktrace by location stacktrace end marker.
# If not, bail out.
if not has_marker(stacktrace, STACKTRACE_END_MARKERS):
return False
# Check if have a UBSan error.
if has_ubsan_error(stacktrace):
return True
# Check if have a stacktrace start marker.
if has_marker(stacktrace, STACKTRACE_TOOL_MARKERS):
return True
return False
def is_check_failure_crash(stacktrace):
"""Return true if it a CHECK failure crash."""
# Android-specific exception patterns.
if environment.is_android():
if 'Device rebooted' in stacktrace:
return True
if 'JNI DETECTED ERROR IN APPLICATION:' in stacktrace:
return True
if re.match(r'.*FATAL EXCEPTION.*:', stacktrace, re.DOTALL):
return True
# FIXME: Analyze why this is not working with chrome.
# If the process has died, it is worthwhile to catch this with even a
# NULL stack.
# process_died_regex = (r'.*Process %s.*\(pid [0-9]+\) has died' %
# environment.get_value('PKG_NAME'))
# if re.match(process_died_regex, stacktrace, re.DOTALL):
# return True
# Application CHECK failure known patterns.
if re.match(r'.*#\s*Fatal error in', stacktrace, re.DOTALL):
return True
if 'Check failed:' in stacktrace:
return True
# Memory debugging tool CHECK failure.
if 'Sanitizer CHECK failed:' in stacktrace:
return True
return False
def filter_binary_path(binary_path):
"""Filters binary path to provide a local copy."""
if environment.is_android():
# Skip symbolization when running it on bad entries like [stack:XYZ].
if not binary_path.startswith('/') or '(deleted)' in binary_path:
return ''
# Initialize some helper variables.
binary_filename = os.path.basename(binary_path)
build_directory = environment.get_value('BUILD_DIR')
symbols_directory = environment.get_value('SYMBOLS_DIR')
# Try to find the library in the build directory first.
local_binary_path = utils.find_binary_path(build_directory,
binary_path)
if local_binary_path:
return local_binary_path
# We didn't find the library locally in the build directory.
# Try finding the library in the local system library cache.
symbols_downloader.download_system_symbols_if_needed(symbols_directory)
local_binary_path = utils.find_binary_path(symbols_directory,
binary_path)
if local_binary_path:
return local_binary_path
# Try pulling in the binary directly from the device into the
# system library cache directory.
local_binary_path = os.path.join(symbols_directory, binary_filename)
adb.run_command('pull %s %s' % (binary_path, local_binary_path))
if os.path.exists(local_binary_path):
return local_binary_path
# Unable to find library.
logs.log_error('Unable to find library %s for symbolization.' %
binary_path)
return ''
if environment.platform() == 'CHROMEOS':
# FIXME: Add code to pull binaries from ChromeOS device.
return binary_path
if environment.is_chromeos_system_job():
# This conditional is True for ChromeOS system fuzzers that are running on
# Linux. Ensure that the binary is always looked for in the chroot and not
# in system directories.
build_dir = environment.get_value('BUILD_DIR')
if not binary_path.startswith(build_dir):
# Fixup path so |binary_path| points to a binary in the chroot (probably
# a system library).
return os.path.join(build_dir, binary_path[1:])
# For Linux and Mac, the binary exists locally. No work to do,
# just return the same binary path.
return binary_path
def get_start_and_end_revision(regression_range, job_type):
"""Get start and end revision."""
start_revision, end_revision = revisions.get_start_and_end_revision(
regression_range)
# FIXME: Hack to use chromium revision for android builds.
if environment.is_android():
return get_chromium_component_start_and_end_revision(
start_revision, end_revision, job_type)
return start_revision, end_revision
def get_gestures(gesture_count):
"""Return a list of random gestures."""
plt = environment.platform()
if environment.is_android(plt):
return android.gestures.get_random_gestures(gesture_count)
if plt == 'LINUX':
return linux.gestures.get_random_gestures(gesture_count)
if plt == 'WINDOWS':
return windows.gestures.get_random_gestures(gesture_count)
return []
def clear_testcase_directories():
"""Clears the testcase directories."""
remove_directory(environment.get_value('FUZZ_INPUTS'), recreate=True)
remove_directory(environment.get_value('FUZZ_INPUTS_DISK'), recreate=True)
if environment.is_android():
from platforms import android
android.device.clear_testcase_directory()
if environment.platform() == 'FUCHSIA':
from platforms import fuchsia
fuchsia.device.clear_testcase_directory()
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
file_host.clear_testcase_directories()
def linkify_kernel_or_lkl_stacktrace_if_needed(crash_info):
"""Linkify Android Kernel or lkl stacktrace."""
kernel_prefix = ''
kernel_hash = ''
if (environment.is_android() and not environment.is_android_emulator() and
(crash_info.found_android_kernel_crash or crash_info.is_kasan)):
kernel_prefix, kernel_hash = \
android_kernel.get_kernel_prefix_and_full_hash()
elif (environment.is_lkl_job() and crash_info.is_lkl
and crash_info.lkl_kernel_build_id):
kernel_prefix, kernel_hash = \
lkl_kernel.get_kernel_prefix_and_full_hash(crash_info.lkl_kernel_build_id)
if kernel_prefix and kernel_hash:
_linkify_android_kernel_stacktrace(crash_info, kernel_prefix,
kernel_hash)
def convert_dependency_url_to_local_path(url):
"""Convert a dependency URL to a corresponding local path."""
# Bot-specific import.
from bot.webserver import http_server
logs.log('Process dependency: %s.' % url)
file_match = FILE_URL_REGEX.search(url)
http_match = HTTP_URL_REGEX.search(url)
platform = environment.platform()
local_path = None
if file_match:
file_path = file_match.group(1)
logs.log('Detected file dependency: %s.' % file_path)
if platform == 'WINDOWS':
local_path = file_path
else:
local_path = '/' + file_path
# Convert remote to local path for android.
if environment.is_android():
remote_testcases_directory = android.constants.DEVICE_TESTCASES_DIR
local_testcases_directory = environment.get_value(
'FUZZ_INPUTS')
local_path = local_path.replace(remote_testcases_directory,
local_testcases_directory)
elif http_match:
relative_http_path = os.path.sep + http_match.group(2)
logs.log('Detected http dependency: %s.' % relative_http_path)
local_path = http_server.get_absolute_testcase_file(relative_http_path)
if not local_path:
# This needs to be a warning since in many cases, it is actually a
# non-existent path. For others, we need to add the directory aliases in
# file http_server.py.
logs.log_warn('Unable to find server resource %s, skipping.' %
relative_http_path)
if local_path:
local_path = utils.normalize_path(local_path)
return local_path
def run_platform_init_scripts():
"""Run platform specific initialization scripts."""
logs.log('Running platform initialization scripts.')
plt = environment.platform()
if environment.is_android():
android_init.run()
elif plt == 'CHROMEOS':
chromeos_init.run()
elif plt == 'FUCHSIA':
fuchsia_init.run()
elif plt == 'LINUX':
linux_init.run()
elif plt == 'MAC':
mac_init.run()
elif plt == 'WINDOWS':
windows_init.run()
else:
raise RuntimeError('Unsupported platform')
logs.log('Completed running platform initialization scripts.')
def get_crash_info_and_stacktrace(application_command_line, crash_stacktrace,
gestures):
"""Return crash minidump location and updated crash stacktrace."""
app_name_lower = environment.get_value('APP_NAME').lower()
retry_limit = environment.get_value('FAIL_RETRIES')
using_android = environment.is_android()
using_chrome = 'chrome' in app_name_lower or 'chromium' in app_name_lower
warmup_timeout = environment.get_value('WARMUP_TIMEOUT', 90)
# Minidump generation is only applicable on Chrome application.
# FIXME: Support minidump generation on platforms other than Android.
if not using_chrome or not using_android:
return None, crash_stacktrace
# Get the crash info from stacktrace.
crash_info = get_crash_info(crash_stacktrace)
# If we lost the minidump file, we need to recreate it.
# Note that because of the way crash_info is generated now, if we have a
# non-None crash_info, we should also have its minidump path; we insert
# the check to safeguard against possibly constructing the crash_info in
# other ways in the future that might potentially lose the minidump path.
if not crash_info or not crash_info.minidump_info.path:
for _ in range(retry_limit):
_, _, output = (
process_handler.run_process(
application_command_line,
timeout=warmup_timeout,
gestures=gestures))
crash_info = get_crash_info(output)
if crash_info and crash_info.minidump_info.path:
crash_stacktrace = utils.decode_to_unicode(output)
break
if not crash_info or not crash_info.minidump_info.path:
# We could not regenerate a minidump for this crash.
logs.log('Unable to regenerate a minidump for this crash.')
return crash_info, crash_stacktrace
def filter_binary_path(binary_path):
"""Filters binary path to provide a local copy."""
if environment.is_android() or environment.is_lkl_job():
return symbols_downloader.filter_binary_path(binary_path)
if environment.platform() == 'CHROMEOS':
# FIXME: Add code to pull binaries from ChromeOS device.
return binary_path
if environment.is_chromeos_system_job():
# This conditional is True for ChromeOS system fuzzers that are running on
# Linux. Ensure that the binary is always looked for in the chroot and not
# in system directories.
build_dir = environment.get_value('BUILD_DIR')
if not binary_path.startswith(build_dir):
# Fixup path so |binary_path| points to a binary in the chroot (probably
# a system library).
return os.path.join(build_dir, binary_path[1:])
# For Linux and Mac, the binary exists locally. No work to do,
# just return the same binary path.
return binary_path
def prepare_log_for_upload(symbolized_output, return_code):
"""Prepare log for upload."""
# Add revision information to the logs.
app_revision = environment.get_value('APP_REVISION')
job_name = environment.get_value('JOB_NAME')
components = revisions.get_component_list(app_revision, job_name)
component_revisions = (revisions.format_revision_list(components,
use_html=False)
or 'Not available.\n')
revisions_header =\
f'Component revisions (build r{app_revision}):\n{component_revisions}\n'
bot_name = environment.get_value('BOT_NAME')
bot_header = f'Bot name: {bot_name}\n'
if environment.is_android():
bot_header += f'Device serial: {environment.get_value("ANDROID_SERIAL")}\n'
return_code_header = "Return code: %s\n\n" % return_code
result = revisions_header + bot_header + return_code_header +\
symbolized_output
return result.encode('utf-8')
def get_command_line_for_application(file_to_run='',
user_profile_index=0,
app_path=None,
app_args=None,
needs_http=False,
write_command_line_file=False,
get_arguments_only=False):
"""Returns the complete command line required to execute application."""
if app_args is None:
app_args = environment.get_value('APP_ARGS')
if app_path is None:
app_path = environment.get_value('APP_PATH')
if not app_path:
# No APP_PATH is available for e.g. grey box fuzzers.
return ''
additional_command_line_flags = get_additional_command_line_flags(
file_to_run)
app_args_append_testcase = environment.get_value(
'APP_ARGS_APPEND_TESTCASE')
app_directory = environment.get_value('APP_DIR')
app_name = environment.get_value('APP_NAME')
apps_argument = environment.get_value('APPS_ARG')
crash_stacks_directory = environment.get_value('CRASH_STACKTRACES_DIR')
debugger = environment.get_value('DEBUGGER_PATH')
device_testcases_directory = android.constants.DEVICE_TESTCASES_DIR
fuzzer_directory = environment.get_value('FUZZER_DIR')
extension_argument = environment.get_value('EXTENSION_ARG')
input_directory = environment.get_value('INPUT_DIR')
launcher = environment.get_value('LAUNCHER_PATH')
is_android = environment.is_android()
root_directory = environment.get_value('ROOT_DIR')
temp_directory = environment.get_value('BOT_TMPDIR')
user_profile_argument = environment.get_value('USER_PROFILE_ARG')
window_argument = environment.get_value('WINDOW_ARG')
user_profile_directory = get_user_profile_directory(user_profile_index)
# Create user profile directory and setup contents if needed.
setup_user_profile_directory_if_needed(user_profile_directory)
# Handle spaces in APP_PATH.
# If application path has spaces, then we need to quote it.
if ' ' in app_path:
app_path = '"%s"' % app_path
interpreter = shell.get_interpreter(app_name)
if get_arguments_only:
# If we are only returning the arguments, do not return the application
# path or anything else required to run it such as an interpreter.
app_path = ''
elif interpreter:
# Prepend command with interpreter if it is a script.
app_path = '%s %s' % (interpreter, app_path)
# Start creating the command line.
command = ''
# Rebase the file_to_run and launcher paths to the worker's root.
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
file_to_run = file_host.rebase_to_worker_root(file_to_run)
launcher = file_host.rebase_to_worker_root(launcher)
# Default case.
testcase_path = file_to_run
testcase_filename = os.path.basename(testcase_path)
testcase_directory = os.path.dirname(testcase_path)
testcase_file_url = utils.file_path_to_file_url(testcase_path)
testcase_http_url = ''
# Determine where |testcase_file_url| should point depending on platform and
# whether or not a launcher script is used.
if file_to_run:
if launcher:
# In the case of launcher scripts, the testcase file to be run resides on
# the host running the launcher script. Thus |testcase_file_url|, which
# may point to a location on the device for Android job types, does not
# apply. Instead, the launcher script should be passed the original file
# to run. By setting |testcase_file_url| to |file_to_run|, we avoid
# duplicating job definitions solely for supporting launcher scripts.
testcase_file_url = file_to_run
# Jobs that have a launcher script which needs to be run on the host will
# have app_name == launcher. In this case don't prepend launcher to
# command - just use app_name.
if os.path.basename(launcher) != app_name:
launcher_with_interpreter = shell.get_execute_command(launcher)
command += launcher_with_interpreter + ' '
elif is_android:
# Android-specific testcase path fixup for fuzzers that don't rely on
# launcher scripts.
local_testcases_directory = environment.get_value('FUZZ_INPUTS')
# Check if the file to run is in fuzzed testcases folder. If yes, then we
# can substitute with a local device path. Otherwise, it is part of some
# data bundle with resource dependencies and we just need to use http
# host forwarder for that.
if file_to_run.startswith(local_testcases_directory):
testcase_relative_path = (
file_to_run[len(local_testcases_directory) + 1:])
testcase_path = os.path.join(device_testcases_directory,
testcase_relative_path)
testcase_file_url = utils.file_path_to_file_url(testcase_path)
else:
# Force use of host_forwarder based on comment above.
needs_http = True
# Check if the testcase needs to be loaded over http.
# TODO(ochang): Make this work for trusted/untrusted.
http_ip = '127.0.0.1'
http_port_1 = environment.get_value('HTTP_PORT_1', 8000)
relative_testcase_path = file_to_run[len(input_directory +
os.path.sep):]
relative_testcase_path = relative_testcase_path.replace('\\', '/')
testcase_http_url = 'http://%s:%d/%s' % (http_ip, http_port_1,
relative_testcase_path)
if needs_http:
# TODO(unassigned): Support https.
testcase_file_url = testcase_http_url
testcase_path = testcase_http_url
# Compose app arguments.
all_app_args = ''
if user_profile_argument:
all_app_args += ' %s=%s' % (user_profile_argument,
user_profile_directory)
if extension_argument and EXTENSIONS_PREFIX in testcase_filename:
all_app_args += ' %s=%s' % (extension_argument, testcase_directory)
if apps_argument and APPS_PREFIX in testcase_filename:
all_app_args += ' %s=%s' % (apps_argument, testcase_directory)
if window_argument:
all_app_args += ' %s' % window_argument
if additional_command_line_flags:
all_app_args += ' %s' % additional_command_line_flags.strip()
if app_args:
all_app_args += ' %s' % app_args.strip()
# Append %TESTCASE% at end if no testcase pattern is found in app arguments.
if not utils.sub_string_exists_in(
['%TESTCASE%', '%TESTCASE_FILE_URL%', '%TESTCASE_HTTP_URL%'],
all_app_args) and app_args_append_testcase:
all_app_args += ' %TESTCASE%'
all_app_args = all_app_args.strip()
# Build the actual command to run now.
if debugger:
command += '%s ' % debugger
if app_path:
command += app_path
if all_app_args:
command += ' %s' % all_app_args
command = command.replace('%APP_DIR%', app_directory)
command = command.replace('%CRASH_STACKTRACES_DIR%',
crash_stacks_directory)
command = command.replace('%DEVICE_TESTCASES_DIR%',
device_testcases_directory)
command = command.replace('%FUZZER_DIR%', fuzzer_directory)
command = command.replace('%INPUT_DIR%', input_directory)
command = command.replace('%ROOT_DIR%', root_directory)
command = command.replace('%TESTCASE%', testcase_path)
command = command.replace('%TESTCASE_FILE_URL%', testcase_file_url)
command = command.replace('%TESTCASE_HTTP_URL%', testcase_http_url)
command = command.replace('%TMP_DIR%', temp_directory)
command = command.replace('%USER_PROFILE_DIR%', user_profile_directory)
if is_android and not launcher:
# Initial setup phase for command line.
if write_command_line_file:
android.adb.write_command_line_file(command, app_path)
return android.app.get_launch_command(all_app_args, testcase_path,
testcase_file_url)
# Decide which directory we will run the application from.
# We are using |app_directory| since it helps to locate pdbs
# in same directory, other dependencies, etc.
if os.path.exists(app_directory):
os.chdir(app_directory)
return str(command)
def terminate_stale_application_instances():
"""Kill stale instances of the application running for this command."""
if environment.is_trusted_host():
from bot.untrusted_runner import remote_process_host
remote_process_host.terminate_stale_application_instances()
return
# Stale instance cleanup is sometimes disabled for local testing.
if not environment.get_value('KILL_STALE_INSTANCES', True):
return
additional_process_to_kill = environment.get_value(
'ADDITIONAL_PROCESSES_TO_KILL')
builds_directory = environment.get_value('BUILDS_DIR')
llvm_symbolizer_filename = environment.get_executable_filename(
'llvm-symbolizer')
platform = environment.platform()
start_time = time.time()
processes_to_kill = []
# Avoid killing the test binary when running the reproduce tool. It is
# commonly in-use on the side on developer workstations.
if not environment.get_value('REPRODUCE_TOOL'):
app_name = environment.get_value('APP_NAME')
processes_to_kill += [app_name]
if additional_process_to_kill:
processes_to_kill += additional_process_to_kill.split(' ')
processes_to_kill = [x for x in processes_to_kill if x]
if environment.is_android(platform):
# Cleanup any stale adb connections.
device_serial = environment.get_value('ANDROID_SERIAL')
adb_search_string = 'adb -s %s' % device_serial
# Terminate llvm symbolizer processes matching exact path. This is important
# for Android where multiple device instances run on same host.
llvm_symbolizer_path = environment.get_llvm_symbolizer_path()
terminate_processes_matching_cmd_line(
[adb_search_string, llvm_symbolizer_path], kill=True)
# Make sure device is online and rooted.
android.adb.run_as_root()
# Make sure to reset SE Linux Permissive Mode (might be lost in reboot).
android.settings.change_se_linux_to_permissive_mode()
# Make sure that device forwarder is running (might be lost in reboot or
# process crash).
android.device.setup_host_and_device_forwarder_if_needed()
# Make sure that package optimization is complete (might be triggered due to
# unexpected circumstances).
android.app.wait_until_optimization_complete()
# Reset application state, which kills its pending instances and re-grants
# the storage permissions.
android.app.reset()
elif platform == 'WINDOWS':
processes_to_kill += [
'cdb.exe',
'handle.exe',
'msdt.exe',
'openwith.exe',
'WerFault.exe',
llvm_symbolizer_filename,
]
terminate_processes_matching_names(processes_to_kill, kill=True)
terminate_processes_matching_cmd_line(builds_directory, kill=True)
# Artifical sleep to let the processes get terminated.
time.sleep(1)
else:
# Handle Linux and Mac platforms.
processes_to_kill += [
'addr2line',
'atos',
'chrome-devel-sandbox',
'gdb',
'nacl_helper',
'xdotool',
llvm_symbolizer_filename,
]
terminate_processes_matching_names(processes_to_kill, kill=True)
terminate_processes_matching_cmd_line(builds_directory, kill=True)
duration = int(time.time() - start_time)
if duration >= 5:
logs.log('Process kill took longer than usual - %s.' %
str(datetime.timedelta(seconds=duration)))
def run_process(cmdline,
current_working_directory=None,
timeout=DEFAULT_TEST_TIMEOUT,
need_shell=False,
gestures=None,
env_copy=None,
testcase_run=True,
ignore_children=True):
"""Executes a process with a given command line and other parameters."""
if environment.is_trusted_host() and testcase_run:
from bot.untrusted_runner import remote_process_host
return remote_process_host.run_process(cmdline,
current_working_directory,
timeout, need_shell, gestures,
env_copy, testcase_run,
ignore_children)
if gestures is None:
gestures = []
if env_copy:
os.environ.update(env_copy)
# FIXME(mbarbella): Using LAUNCHER_PATH here is error prone. It forces us to
# do certain operations before fuzzer setup (e.g. bad build check).
launcher = environment.get_value('LAUNCHER_PATH')
# This is used when running scripts on native linux OS and not on the device.
# E.g. running a fuzzer to generate testcases or launcher script.
plt = environment.platform()
is_android = environment.is_android(plt)
runs_on_device = is_android or plt == 'FUCHSIA'
if runs_on_device and (not testcase_run or launcher):
plt = 'LINUX'
# Lower down testcase timeout slightly to account for time for crash analysis.
timeout -= CRASH_ANALYSIS_TIME
# LeakSanitizer hack - give time for stdout/stderr processing.
lsan = environment.get_value('LSAN', False)
if lsan:
timeout -= LSAN_ANALYSIS_TIME
# Initialize variables.
adb_output = None
process_output = ''
process_status = None
return_code = 0
process_poll_interval = environment.get_value('PROCESS_POLL_INTERVAL', 0.5)
start_time = time.time()
watch_for_process_exit = (environment.get_value('WATCH_FOR_PROCESS_EXIT')
if is_android else True)
window_list = []
# Get gesture start time from last element in gesture list.
gestures = copy.deepcopy(gestures)
if gestures and gestures[-1].startswith('Trigger'):
gesture_start_time = int(gestures[-1].split(':')[1])
gestures.pop()
else:
gesture_start_time = timeout // 2
if is_android:
# Clear the log upfront.
android.logger.clear_log()
# Run the app.
adb_output = android.adb.run_command(cmdline, timeout=timeout)
else:
cmd, args = shell.get_command_and_arguments(cmdline)
process_output = mozprocess.processhandler.StoreOutput()
process_status = ProcessStatus()
try:
process_handle = mozprocess.ProcessHandlerMixin(
cmd,
args,
cwd=current_working_directory,
shell=need_shell,
processOutputLine=[process_output],
onFinish=[process_status],
ignore_children=ignore_children)
start_process(process_handle)
except:
logs.log_error('Exception occurred when running command: %s.' %
cmdline)
return None, None, ''
while True:
time.sleep(process_poll_interval)
# Run the gestures at gesture_start_time or in case we didn't find windows
# in the last try.
if (gestures and time.time() - start_time >= gesture_start_time
and not window_list):
# In case, we don't find any windows, we increment the gesture start time
# so that the next check is after 1 second.
gesture_start_time += 1
if plt == 'LINUX':
linux.gestures.run_gestures(gestures, process_handle.pid,
process_status, start_time,
timeout, window_list)
elif plt == 'WINDOWS':
windows.gestures.run_gestures(gestures, process_handle.pid,
process_status, start_time,
timeout, window_list)
elif is_android:
android.gestures.run_gestures(gestures, start_time, timeout)
# TODO(mbarbella): We add a fake window here to prevent gestures on
# Android from getting executed more than once.
window_list = ['FAKE']
if time.time() - start_time >= timeout:
break
# Collect the process output.
output = (android.logger.log_output()
if is_android else b'\n'.join(process_output.output))
output = utils.decode_to_unicode(output)
if crash_analyzer.is_memory_tool_crash(output):
break
# Check if we need to bail out on process exit.
if watch_for_process_exit:
# If |watch_for_process_exit| is set, then we already completed running
# our app launch command. So, we can bail out.
if is_android:
break
# On desktop, we bail out as soon as the process finishes.
if process_status and process_status.finished:
# Wait for process shutdown and set return code.
process_handle.wait(timeout=PROCESS_CLEANUP_WAIT_TIME)
break
# Process output based on platform.
if is_android:
# Get current log output. If device is in reboot mode, logcat automatically
# waits for device to be online.
time.sleep(ANDROID_CRASH_LOGCAT_WAIT_TIME)
output = android.logger.log_output()
if android.constants.LOW_MEMORY_REGEX.search(output):
# If the device is low on memory, we should force reboot and bail out to
# prevent device from getting in a frozen state.
logs.log('Device is low on memory, rebooting.', output=output)
android.adb.hard_reset()
android.adb.wait_for_device()
elif android.adb.time_since_last_reboot() < time.time() - start_time:
# Check if a reboot has happened, if yes, append log output before reboot
# and kernel logs content to output.
log_before_last_reboot = android.logger.log_output_before_last_reboot(
)
kernel_log = android.adb.get_kernel_log_content()
output = '%s%s%s%s%s' % (
log_before_last_reboot,
utils.get_line_seperator('Device rebooted'), output,
utils.get_line_seperator('Kernel Log'), kernel_log)
# Make sure to reset SE Linux Permissive Mode. This can be done cheaply
# in ~0.15 sec and is needed especially between runs for kernel crashes.
android.adb.run_as_root()
android.settings.change_se_linux_to_permissive_mode()
return_code = 1
# Add output from adb to the front.
if adb_output:
output = '%s\n\n%s' % (adb_output, output)
# Kill the application if it is still running. We do this at the end to
# prevent this from adding noise to the logcat output.
task_name = environment.get_value('TASK_NAME')
child_process_termination_pattern = environment.get_value(
'CHILD_PROCESS_TERMINATION_PATTERN')
if task_name == 'fuzz' and child_process_termination_pattern:
# In some cases, we do not want to terminate the application after each
# run to avoid long startup times (e.g. for chrome). Terminate processes
# matching a particular pattern for light cleanup in this case.
android.adb.kill_processes_and_children_matching_name(
child_process_termination_pattern)
else:
# There is no special termination behavior. Simply stop the application.
android.app.stop()
else:
# Get the return code in case the process has finished already.
# If the process hasn't finished, return_code will be None which is what
# callers expect unless the output indicates a crash.
return_code = process_handle.poll()
# If the process is still running, then terminate it.
if not process_status.finished:
launcher_with_interpreter = shell.get_execute_command(
launcher) if launcher else None
if (launcher_with_interpreter
and cmdline.startswith(launcher_with_interpreter)):
# If this was a launcher script, we KILL all child processes created
# except for APP_NAME.
# It is expected that, if the launcher script terminated normally, it
# cleans up all the child processes it created itself.
terminate_root_and_child_processes(process_handle.pid)
else:
try:
# kill() here actually sends SIGTERM on posix.
process_handle.kill()
except:
pass
if lsan:
time.sleep(LSAN_ANALYSIS_TIME)
output = b'\n'.join(process_output.output)
output = utils.decode_to_unicode(output)
# X Server hack when max client reached.
if ('Maximum number of clients reached' in output
or 'Unable to get connection to X server' in output):
logs.log_error('Unable to connect to X server, exiting.')
os.system('sudo killall -9 Xvfb blackbox >/dev/null 2>&1')
sys.exit(0)
if testcase_run and (crash_analyzer.is_memory_tool_crash(output)
or crash_analyzer.is_check_failure_crash(output)):
return_code = 1
# If a crash is found, then we add the memory state as well.
if return_code and is_android:
ps_output = android.adb.get_ps_output()
if ps_output:
output += utils.get_line_seperator('Memory Statistics')
output += ps_output
if return_code:
logs.log_warn('Process (%s) ended with exit code (%s).' %
(repr(cmdline), str(return_code)),
output=output)
return return_code, round(time.time() - start_time, 1), output
def clear_device_temp_directories():
"""Clear device specific temp directories."""
if environment.is_android() and environment.get_value('ANDROID_SERIAL'):
from platforms import android
android.device.clear_temp_directories()
def get_crash_info(output):
"""Parse crash output to get (local) minidump path and any other information
useful for crash uploading, and store in a CrashReportInfo object."""
crash_stacks_directory = environment.get_value('CRASH_STACKTRACES_DIR')
output_lines = output.splitlines()
num_lines = len(output_lines)
is_android = environment.is_android()
for i, line in enumerate(output_lines):
if is_android:
# If we are on Android, the dump extraction is more complicated.
# The location placed in the crash-stacktrace is of the dump itself but
# in fact only the MIME of the dump exists, and will have a different
# extension. We need to pull the MIME and process it.
match = re.match(CRASH_DUMP_PATH_MARKER, line)
if not match:
continue
minidump_mime_filename_base = None
for j in range(i + 1, num_lines):
line = output_lines[j]
match = re.match(r'(.*)\.dmp', line)
if match:
minidump_mime_filename_base = os.path.basename(match.group(1).strip())
break
if not minidump_mime_filename_base:
logs.log_error('Minidump marker was found, but no path in stacktrace.')
return None
# Look for MIME. If none found, bail.
# We might not have copied over the crash dumps yet (copying is buffered),
# so we want to search both the original directory and the one to which
# the minidumps should later be copied.
device_directories_to_search = [
constants.CRASH_DUMPS_DIR,
os.path.dirname(line.strip())
]
device_minidump_search_paths = []
device_minidump_mime_path = None
for device_directory in device_directories_to_search:
device_minidump_mime_potential_paths = adb.run_shell_command(
['ls', '"%s"' % device_directory], root=True).splitlines()
device_minidump_search_paths += device_minidump_mime_potential_paths
for potential_path in device_minidump_mime_potential_paths:
# Check that we actually found a file, and the right one (not logcat).
if 'No such file or directory' in potential_path:
continue
if minidump_mime_filename_base not in potential_path:
continue
if '.up' in potential_path or '.dmp' in potential_path:
device_minidump_mime_path = os.path.join(device_directory,
potential_path)
break
# Break if we found a path.
if device_minidump_mime_path is not None:
break
# If we still didn't find a minidump path, bail.
if device_minidump_mime_path is None:
logs.log_error('Could not get MIME path from ls:\n%s' %
str(device_minidump_search_paths))
return None
# Pull out MIME and parse to minidump file and MIME parameters.
minidump_mime_filename = '%s.mime' % minidump_mime_filename_base
local_minidump_mime_path = os.path.join(crash_stacks_directory,
minidump_mime_filename)
adb.run_command([
'pull',
'"%s"' % device_minidump_mime_path, local_minidump_mime_path
])
if not os.path.exists(local_minidump_mime_path):
logs.log_error('Could not pull MIME from %s to %s.' %
(device_minidump_mime_path, local_minidump_mime_path))
return None
crash_info = parse_mime_to_crash_report_info(local_minidump_mime_path)
if crash_info is None:
return None
crash_info.unsymbolized_stacktrace = output
return crash_info
# Other platforms are not currently supported.
logs.log_error('Unable to fetch crash information for this platform.')
return None
# Could not find dump location, bail out. This could also happen when we don't
# have a minidump location in stack at all, e.g. when testcase does not crash
# during minimization.
return None
def clear_device_temp_directories():
"""Clear device specific temp directories."""
if environment.is_android():
from platforms import android
android.device.clear_temp_directories()
def setup_testcase(testcase, job_type, fuzzer_override=None):
"""Sets up the testcase and needed dependencies like fuzzer,
data bundle, etc."""
fuzzer_name = fuzzer_override or testcase.fuzzer_name
task_name = environment.get_value('TASK_NAME')
testcase_fail_wait = environment.get_value('FAIL_WAIT')
testcase_id = testcase.key.id()
# Clear testcase directories.
shell.clear_testcase_directories()
# Adjust the test timeout value if this is coming from an user uploaded
# testcase.
if testcase.uploader_email:
_set_timeout_value_from_user_upload(testcase_id)
# Update the fuzzer if necessary in order to get the updated data bundle.
if fuzzer_name:
try:
update_successful = update_fuzzer_and_data_bundles(fuzzer_name)
except errors.InvalidFuzzerError:
# Close testcase and don't recreate tasks if this fuzzer is invalid.
testcase.open = False
testcase.fixed = 'NA'
testcase.set_metadata('fuzzer_was_deleted', True)
logs.log_error('Closed testcase %d with invalid fuzzer %s.' %
(testcase_id, fuzzer_name))
error_message = 'Fuzzer %s no longer exists' % fuzzer_name
data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR,
error_message)
return None, None, None
if not update_successful:
error_message = 'Unable to setup fuzzer %s' % fuzzer_name
data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR,
error_message)
tasks.add_task(
task_name, testcase_id, job_type, wait_time=testcase_fail_wait)
return None, None, None
# Extract the testcase and any of its resources to the input directory.
file_list, input_directory, testcase_file_path = unpack_testcase(testcase)
if not file_list:
error_message = 'Unable to setup testcase %s' % testcase_file_path
data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR,
error_message)
tasks.add_task(
task_name, testcase_id, job_type, wait_time=testcase_fail_wait)
return None, None, None
# For Android/Fuchsia, we need to sync our local testcases directory with the
# one on the device.
if environment.is_android():
_copy_testcase_to_device_and_setup_environment(testcase, testcase_file_path)
# Push testcases to worker.
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
file_host.push_testcases_to_worker()
# Copy global blacklist into local blacklist.
is_lsan_enabled = environment.get_value('LSAN')
if is_lsan_enabled:
# Get local blacklist without this testcase's entry.
leak_blacklist.copy_global_to_local_blacklist(excluded_testcase=testcase)
prepare_environment_for_testcase(testcase, job_type, task_name)
return file_list, input_directory, testcase_file_path
def get_crash_data(crash_data,
symbolize_flag=True,
fuzz_target=None,
already_symbolized=False,
detect_ooms_and_hangs=None):
"""Get crash parameters from crash data.
Crash parameters include crash type, address, state and stacktrace.
If the stacktrace is not already symbolized, we will try to symbolize it
unless |symbolize| flag is set to False. Symbolized stacktrace will contain
inline frames, but we do exclude them for purposes of crash state generation
(helps in testcase deduplication)."""
# Decide whether to symbolize or not symbolize the input stacktrace.
# Note that Fuchsia logs are always symbolized.
if symbolize_flag:
# Defer imports since stack_symbolizer pulls in a lot of things.
from crash_analysis.stack_parsing import stack_symbolizer
crash_stacktrace_with_inlines = stack_symbolizer.symbolize_stacktrace(
crash_data, enable_inline_frames=True)
crash_stacktrace_without_inlines = stack_symbolizer.symbolize_stacktrace(
crash_data, enable_inline_frames=False)
else:
# We are explicitly indicated to not symbolize using |symbolize_flag|. There
# is no distinction between inline and non-inline frames for an unsymbolized
# stacktrace.
crash_stacktrace_with_inlines = crash_data
crash_stacktrace_without_inlines = crash_data
# Additional stack frame ignore regexes.
custom_stack_frame_ignore_regexes = (local_config.ProjectConfig().get(
'stacktrace.stack_frame_ignore_regexes', []))
if environment.get_value('TASK_NAME') == 'analyze':
detect_v8_runtime_errors = True
else:
detect_v8_runtime_errors = environment.get_value(
'DETECT_V8_RUNTIME_ERRORS', False)
fuzz_target = fuzz_target or environment.get_value('FUZZ_TARGET')
redzone_size = environment.get_value('REDZONE')
if detect_ooms_and_hangs is None:
detect_ooms_and_hangs = (
environment.get_value('REPORT_OOMS_AND_HANGS')
and (not redzone_size
or redzone_size <= MAX_REDZONE_SIZE_FOR_OOMS_AND_HANGS))
include_ubsan = 'halt_on_error=0' not in environment.get_value(
'UBSAN_OPTIONS', '')
stack_parser = stacktraces.StackParser(
symbolized=symbolize_flag or already_symbolized,
detect_ooms_and_hangs=detect_ooms_and_hangs,
detect_v8_runtime_errors=detect_v8_runtime_errors,
custom_stack_frame_ignore_regexes=custom_stack_frame_ignore_regexes,
fuzz_target=fuzz_target,
include_ubsan=include_ubsan)
result = stack_parser.parse(crash_stacktrace_without_inlines)
# Use stacktrace with inlines for the result.
if result.crash_stacktrace:
result.crash_stacktrace = crash_stacktrace_with_inlines
# Linkify Android stacktrace.
if environment.is_android() and (result.found_android_kernel_crash
or result.is_kasan):
linkify_android_stacktrace(result)
return result
