def is_memory_tool_crash(stacktrace): """Return true if it is a memory debugging tool crash.""" # Job-specific generic checks. crash_signature = environment.get_value('CRASH_SIGNATURE') if crash_signature and re.search(crash_signature, stacktrace): return True # Android specific check. # FIXME: Share this regex with stack_analyzer. if (environment.is_android() and re.match( r'.*signal.*\(SIG.*fault addr ([^ ]*)', stacktrace, re.DOTALL)): return True # Check if we have a complete stacktrace by location stacktrace end marker. # If not, bail out. if not has_marker(stacktrace, STACKTRACE_END_MARKERS): return False # Check if have a UBSan error. if has_ubsan_error(stacktrace): return True # Check if have a stacktrace start marker. if has_marker(stacktrace, STACKTRACE_TOOL_MARKERS): return True return False
def is_check_failure_crash(stacktrace): """Return true if it a CHECK failure crash.""" # Android-specific exception patterns. if environment.is_android(): if 'Device rebooted' in stacktrace: return True if 'JNI DETECTED ERROR IN APPLICATION:' in stacktrace: return True if re.match(r'.*FATAL EXCEPTION.*:', stacktrace, re.DOTALL): return True # FIXME: Analyze why this is not working with chrome. # If the process has died, it is worthwhile to catch this with even a # NULL stack. # process_died_regex = (r'.*Process %s.*\(pid [0-9]+\) has died' % # environment.get_value('PKG_NAME')) # if re.match(process_died_regex, stacktrace, re.DOTALL): # return True # Application CHECK failure known patterns. if re.match(r'.*#\s*Fatal error in', stacktrace, re.DOTALL): return True if 'Check failed:' in stacktrace: return True # Memory debugging tool CHECK failure. if 'Sanitizer CHECK failed:' in stacktrace: return True return False
def filter_binary_path(binary_path): """Filters binary path to provide a local copy.""" if environment.is_android(): # Skip symbolization when running it on bad entries like [stack:XYZ]. if not binary_path.startswith('/') or '(deleted)' in binary_path: return '' # Initialize some helper variables. binary_filename = os.path.basename(binary_path) build_directory = environment.get_value('BUILD_DIR') symbols_directory = environment.get_value('SYMBOLS_DIR') # Try to find the library in the build directory first. local_binary_path = utils.find_binary_path(build_directory, binary_path) if local_binary_path: return local_binary_path # We didn't find the library locally in the build directory. # Try finding the library in the local system library cache. symbols_downloader.download_system_symbols_if_needed(symbols_directory) local_binary_path = utils.find_binary_path(symbols_directory, binary_path) if local_binary_path: return local_binary_path # Try pulling in the binary directly from the device into the # system library cache directory. local_binary_path = os.path.join(symbols_directory, binary_filename) adb.run_command('pull %s %s' % (binary_path, local_binary_path)) if os.path.exists(local_binary_path): return local_binary_path # Unable to find library. logs.log_error('Unable to find library %s for symbolization.' % binary_path) return '' if environment.platform() == 'CHROMEOS': # FIXME: Add code to pull binaries from ChromeOS device. return binary_path if environment.is_chromeos_system_job(): # This conditional is True for ChromeOS system fuzzers that are running on # Linux. Ensure that the binary is always looked for in the chroot and not # in system directories. build_dir = environment.get_value('BUILD_DIR') if not binary_path.startswith(build_dir): # Fixup path so |binary_path| points to a binary in the chroot (probably # a system library). return os.path.join(build_dir, binary_path[1:]) # For Linux and Mac, the binary exists locally. No work to do, # just return the same binary path. return binary_path
def get_start_and_end_revision(regression_range, job_type): """Get start and end revision.""" start_revision, end_revision = revisions.get_start_and_end_revision( regression_range) # FIXME: Hack to use chromium revision for android builds. if environment.is_android(): return get_chromium_component_start_and_end_revision( start_revision, end_revision, job_type) return start_revision, end_revision
def get_gestures(gesture_count): """Return a list of random gestures.""" plt = environment.platform() if environment.is_android(plt): return android.gestures.get_random_gestures(gesture_count) if plt == 'LINUX': return linux.gestures.get_random_gestures(gesture_count) if plt == 'WINDOWS': return windows.gestures.get_random_gestures(gesture_count) return []
def clear_testcase_directories(): """Clears the testcase directories.""" remove_directory(environment.get_value('FUZZ_INPUTS'), recreate=True) remove_directory(environment.get_value('FUZZ_INPUTS_DISK'), recreate=True) if environment.is_android(): from platforms import android android.device.clear_testcase_directory() if environment.platform() == 'FUCHSIA': from platforms import fuchsia fuchsia.device.clear_testcase_directory() if environment.is_trusted_host(): from bot.untrusted_runner import file_host file_host.clear_testcase_directories()
def linkify_kernel_or_lkl_stacktrace_if_needed(crash_info): """Linkify Android Kernel or lkl stacktrace.""" kernel_prefix = '' kernel_hash = '' if (environment.is_android() and not environment.is_android_emulator() and (crash_info.found_android_kernel_crash or crash_info.is_kasan)): kernel_prefix, kernel_hash = \ android_kernel.get_kernel_prefix_and_full_hash() elif (environment.is_lkl_job() and crash_info.is_lkl and crash_info.lkl_kernel_build_id): kernel_prefix, kernel_hash = \ lkl_kernel.get_kernel_prefix_and_full_hash(crash_info.lkl_kernel_build_id) if kernel_prefix and kernel_hash: _linkify_android_kernel_stacktrace(crash_info, kernel_prefix, kernel_hash)
def convert_dependency_url_to_local_path(url): """Convert a dependency URL to a corresponding local path.""" # Bot-specific import. from bot.webserver import http_server logs.log('Process dependency: %s.' % url) file_match = FILE_URL_REGEX.search(url) http_match = HTTP_URL_REGEX.search(url) platform = environment.platform() local_path = None if file_match: file_path = file_match.group(1) logs.log('Detected file dependency: %s.' % file_path) if platform == 'WINDOWS': local_path = file_path else: local_path = '/' + file_path # Convert remote to local path for android. if environment.is_android(): remote_testcases_directory = android.constants.DEVICE_TESTCASES_DIR local_testcases_directory = environment.get_value( 'FUZZ_INPUTS') local_path = local_path.replace(remote_testcases_directory, local_testcases_directory) elif http_match: relative_http_path = os.path.sep + http_match.group(2) logs.log('Detected http dependency: %s.' % relative_http_path) local_path = http_server.get_absolute_testcase_file(relative_http_path) if not local_path: # This needs to be a warning since in many cases, it is actually a # non-existent path. For others, we need to add the directory aliases in # file http_server.py. logs.log_warn('Unable to find server resource %s, skipping.' % relative_http_path) if local_path: local_path = utils.normalize_path(local_path) return local_path
def run_platform_init_scripts(): """Run platform specific initialization scripts.""" logs.log('Running platform initialization scripts.') plt = environment.platform() if environment.is_android(): android_init.run() elif plt == 'CHROMEOS': chromeos_init.run() elif plt == 'FUCHSIA': fuchsia_init.run() elif plt == 'LINUX': linux_init.run() elif plt == 'MAC': mac_init.run() elif plt == 'WINDOWS': windows_init.run() else: raise RuntimeError('Unsupported platform') logs.log('Completed running platform initialization scripts.')
def get_crash_info_and_stacktrace(application_command_line, crash_stacktrace, gestures): """Return crash minidump location and updated crash stacktrace.""" app_name_lower = environment.get_value('APP_NAME').lower() retry_limit = environment.get_value('FAIL_RETRIES') using_android = environment.is_android() using_chrome = 'chrome' in app_name_lower or 'chromium' in app_name_lower warmup_timeout = environment.get_value('WARMUP_TIMEOUT', 90) # Minidump generation is only applicable on Chrome application. # FIXME: Support minidump generation on platforms other than Android. if not using_chrome or not using_android: return None, crash_stacktrace # Get the crash info from stacktrace. crash_info = get_crash_info(crash_stacktrace) # If we lost the minidump file, we need to recreate it. # Note that because of the way crash_info is generated now, if we have a # non-None crash_info, we should also have its minidump path; we insert # the check to safeguard against possibly constructing the crash_info in # other ways in the future that might potentially lose the minidump path. if not crash_info or not crash_info.minidump_info.path: for _ in range(retry_limit): _, _, output = ( process_handler.run_process( application_command_line, timeout=warmup_timeout, gestures=gestures)) crash_info = get_crash_info(output) if crash_info and crash_info.minidump_info.path: crash_stacktrace = utils.decode_to_unicode(output) break if not crash_info or not crash_info.minidump_info.path: # We could not regenerate a minidump for this crash. logs.log('Unable to regenerate a minidump for this crash.') return crash_info, crash_stacktrace
def filter_binary_path(binary_path): """Filters binary path to provide a local copy.""" if environment.is_android() or environment.is_lkl_job(): return symbols_downloader.filter_binary_path(binary_path) if environment.platform() == 'CHROMEOS': # FIXME: Add code to pull binaries from ChromeOS device. return binary_path if environment.is_chromeos_system_job(): # This conditional is True for ChromeOS system fuzzers that are running on # Linux. Ensure that the binary is always looked for in the chroot and not # in system directories. build_dir = environment.get_value('BUILD_DIR') if not binary_path.startswith(build_dir): # Fixup path so |binary_path| points to a binary in the chroot (probably # a system library). return os.path.join(build_dir, binary_path[1:]) # For Linux and Mac, the binary exists locally. No work to do, # just return the same binary path. return binary_path
def prepare_log_for_upload(symbolized_output, return_code): """Prepare log for upload.""" # Add revision information to the logs. app_revision = environment.get_value('APP_REVISION') job_name = environment.get_value('JOB_NAME') components = revisions.get_component_list(app_revision, job_name) component_revisions = (revisions.format_revision_list(components, use_html=False) or 'Not available.\n') revisions_header =\ f'Component revisions (build r{app_revision}):\n{component_revisions}\n' bot_name = environment.get_value('BOT_NAME') bot_header = f'Bot name: {bot_name}\n' if environment.is_android(): bot_header += f'Device serial: {environment.get_value("ANDROID_SERIAL")}\n' return_code_header = "Return code: %s\n\n" % return_code result = revisions_header + bot_header + return_code_header +\ symbolized_output return result.encode('utf-8')
def get_command_line_for_application(file_to_run='', user_profile_index=0, app_path=None, app_args=None, needs_http=False, write_command_line_file=False, get_arguments_only=False): """Returns the complete command line required to execute application.""" if app_args is None: app_args = environment.get_value('APP_ARGS') if app_path is None: app_path = environment.get_value('APP_PATH') if not app_path: # No APP_PATH is available for e.g. grey box fuzzers. return '' additional_command_line_flags = get_additional_command_line_flags( file_to_run) app_args_append_testcase = environment.get_value( 'APP_ARGS_APPEND_TESTCASE') app_directory = environment.get_value('APP_DIR') app_name = environment.get_value('APP_NAME') apps_argument = environment.get_value('APPS_ARG') crash_stacks_directory = environment.get_value('CRASH_STACKTRACES_DIR') debugger = environment.get_value('DEBUGGER_PATH') device_testcases_directory = android.constants.DEVICE_TESTCASES_DIR fuzzer_directory = environment.get_value('FUZZER_DIR') extension_argument = environment.get_value('EXTENSION_ARG') input_directory = environment.get_value('INPUT_DIR') launcher = environment.get_value('LAUNCHER_PATH') is_android = environment.is_android() root_directory = environment.get_value('ROOT_DIR') temp_directory = environment.get_value('BOT_TMPDIR') user_profile_argument = environment.get_value('USER_PROFILE_ARG') window_argument = environment.get_value('WINDOW_ARG') user_profile_directory = get_user_profile_directory(user_profile_index) # Create user profile directory and setup contents if needed. setup_user_profile_directory_if_needed(user_profile_directory) # Handle spaces in APP_PATH. # If application path has spaces, then we need to quote it. if ' ' in app_path: app_path = '"%s"' % app_path interpreter = shell.get_interpreter(app_name) if get_arguments_only: # If we are only returning the arguments, do not return the application # path or anything else required to run it such as an interpreter. app_path = '' elif interpreter: # Prepend command with interpreter if it is a script. app_path = '%s %s' % (interpreter, app_path) # Start creating the command line. command = '' # Rebase the file_to_run and launcher paths to the worker's root. if environment.is_trusted_host(): from bot.untrusted_runner import file_host file_to_run = file_host.rebase_to_worker_root(file_to_run) launcher = file_host.rebase_to_worker_root(launcher) # Default case. testcase_path = file_to_run testcase_filename = os.path.basename(testcase_path) testcase_directory = os.path.dirname(testcase_path) testcase_file_url = utils.file_path_to_file_url(testcase_path) testcase_http_url = '' # Determine where |testcase_file_url| should point depending on platform and # whether or not a launcher script is used. if file_to_run: if launcher: # In the case of launcher scripts, the testcase file to be run resides on # the host running the launcher script. Thus |testcase_file_url|, which # may point to a location on the device for Android job types, does not # apply. Instead, the launcher script should be passed the original file # to run. By setting |testcase_file_url| to |file_to_run|, we avoid # duplicating job definitions solely for supporting launcher scripts. testcase_file_url = file_to_run # Jobs that have a launcher script which needs to be run on the host will # have app_name == launcher. In this case don't prepend launcher to # command - just use app_name. if os.path.basename(launcher) != app_name: launcher_with_interpreter = shell.get_execute_command(launcher) command += launcher_with_interpreter + ' ' elif is_android: # Android-specific testcase path fixup for fuzzers that don't rely on # launcher scripts. local_testcases_directory = environment.get_value('FUZZ_INPUTS') # Check if the file to run is in fuzzed testcases folder. If yes, then we # can substitute with a local device path. Otherwise, it is part of some # data bundle with resource dependencies and we just need to use http # host forwarder for that. if file_to_run.startswith(local_testcases_directory): testcase_relative_path = ( file_to_run[len(local_testcases_directory) + 1:]) testcase_path = os.path.join(device_testcases_directory, testcase_relative_path) testcase_file_url = utils.file_path_to_file_url(testcase_path) else: # Force use of host_forwarder based on comment above. needs_http = True # Check if the testcase needs to be loaded over http. # TODO(ochang): Make this work for trusted/untrusted. http_ip = '127.0.0.1' http_port_1 = environment.get_value('HTTP_PORT_1', 8000) relative_testcase_path = file_to_run[len(input_directory + os.path.sep):] relative_testcase_path = relative_testcase_path.replace('\\', '/') testcase_http_url = 'http://%s:%d/%s' % (http_ip, http_port_1, relative_testcase_path) if needs_http: # TODO(unassigned): Support https. testcase_file_url = testcase_http_url testcase_path = testcase_http_url # Compose app arguments. all_app_args = '' if user_profile_argument: all_app_args += ' %s=%s' % (user_profile_argument, user_profile_directory) if extension_argument and EXTENSIONS_PREFIX in testcase_filename: all_app_args += ' %s=%s' % (extension_argument, testcase_directory) if apps_argument and APPS_PREFIX in testcase_filename: all_app_args += ' %s=%s' % (apps_argument, testcase_directory) if window_argument: all_app_args += ' %s' % window_argument if additional_command_line_flags: all_app_args += ' %s' % additional_command_line_flags.strip() if app_args: all_app_args += ' %s' % app_args.strip() # Append %TESTCASE% at end if no testcase pattern is found in app arguments. if not utils.sub_string_exists_in( ['%TESTCASE%', '%TESTCASE_FILE_URL%', '%TESTCASE_HTTP_URL%'], all_app_args) and app_args_append_testcase: all_app_args += ' %TESTCASE%' all_app_args = all_app_args.strip() # Build the actual command to run now. if debugger: command += '%s ' % debugger if app_path: command += app_path if all_app_args: command += ' %s' % all_app_args command = command.replace('%APP_DIR%', app_directory) command = command.replace('%CRASH_STACKTRACES_DIR%', crash_stacks_directory) command = command.replace('%DEVICE_TESTCASES_DIR%', device_testcases_directory) command = command.replace('%FUZZER_DIR%', fuzzer_directory) command = command.replace('%INPUT_DIR%', input_directory) command = command.replace('%ROOT_DIR%', root_directory) command = command.replace('%TESTCASE%', testcase_path) command = command.replace('%TESTCASE_FILE_URL%', testcase_file_url) command = command.replace('%TESTCASE_HTTP_URL%', testcase_http_url) command = command.replace('%TMP_DIR%', temp_directory) command = command.replace('%USER_PROFILE_DIR%', user_profile_directory) if is_android and not launcher: # Initial setup phase for command line. if write_command_line_file: android.adb.write_command_line_file(command, app_path) return android.app.get_launch_command(all_app_args, testcase_path, testcase_file_url) # Decide which directory we will run the application from. # We are using |app_directory| since it helps to locate pdbs # in same directory, other dependencies, etc. if os.path.exists(app_directory): os.chdir(app_directory) return str(command)
def terminate_stale_application_instances(): """Kill stale instances of the application running for this command.""" if environment.is_trusted_host(): from bot.untrusted_runner import remote_process_host remote_process_host.terminate_stale_application_instances() return # Stale instance cleanup is sometimes disabled for local testing. if not environment.get_value('KILL_STALE_INSTANCES', True): return additional_process_to_kill = environment.get_value( 'ADDITIONAL_PROCESSES_TO_KILL') builds_directory = environment.get_value('BUILDS_DIR') llvm_symbolizer_filename = environment.get_executable_filename( 'llvm-symbolizer') platform = environment.platform() start_time = time.time() processes_to_kill = [] # Avoid killing the test binary when running the reproduce tool. It is # commonly in-use on the side on developer workstations. if not environment.get_value('REPRODUCE_TOOL'): app_name = environment.get_value('APP_NAME') processes_to_kill += [app_name] if additional_process_to_kill: processes_to_kill += additional_process_to_kill.split(' ') processes_to_kill = [x for x in processes_to_kill if x] if environment.is_android(platform): # Cleanup any stale adb connections. device_serial = environment.get_value('ANDROID_SERIAL') adb_search_string = 'adb -s %s' % device_serial # Terminate llvm symbolizer processes matching exact path. This is important # for Android where multiple device instances run on same host. llvm_symbolizer_path = environment.get_llvm_symbolizer_path() terminate_processes_matching_cmd_line( [adb_search_string, llvm_symbolizer_path], kill=True) # Make sure device is online and rooted. android.adb.run_as_root() # Make sure to reset SE Linux Permissive Mode (might be lost in reboot). android.settings.change_se_linux_to_permissive_mode() # Make sure that device forwarder is running (might be lost in reboot or # process crash). android.device.setup_host_and_device_forwarder_if_needed() # Make sure that package optimization is complete (might be triggered due to # unexpected circumstances). android.app.wait_until_optimization_complete() # Reset application state, which kills its pending instances and re-grants # the storage permissions. android.app.reset() elif platform == 'WINDOWS': processes_to_kill += [ 'cdb.exe', 'handle.exe', 'msdt.exe', 'openwith.exe', 'WerFault.exe', llvm_symbolizer_filename, ] terminate_processes_matching_names(processes_to_kill, kill=True) terminate_processes_matching_cmd_line(builds_directory, kill=True) # Artifical sleep to let the processes get terminated. time.sleep(1) else: # Handle Linux and Mac platforms. processes_to_kill += [ 'addr2line', 'atos', 'chrome-devel-sandbox', 'gdb', 'nacl_helper', 'xdotool', llvm_symbolizer_filename, ] terminate_processes_matching_names(processes_to_kill, kill=True) terminate_processes_matching_cmd_line(builds_directory, kill=True) duration = int(time.time() - start_time) if duration >= 5: logs.log('Process kill took longer than usual - %s.' % str(datetime.timedelta(seconds=duration)))
def run_process(cmdline, current_working_directory=None, timeout=DEFAULT_TEST_TIMEOUT, need_shell=False, gestures=None, env_copy=None, testcase_run=True, ignore_children=True): """Executes a process with a given command line and other parameters.""" if environment.is_trusted_host() and testcase_run: from bot.untrusted_runner import remote_process_host return remote_process_host.run_process(cmdline, current_working_directory, timeout, need_shell, gestures, env_copy, testcase_run, ignore_children) if gestures is None: gestures = [] if env_copy: os.environ.update(env_copy) # FIXME(mbarbella): Using LAUNCHER_PATH here is error prone. It forces us to # do certain operations before fuzzer setup (e.g. bad build check). launcher = environment.get_value('LAUNCHER_PATH') # This is used when running scripts on native linux OS and not on the device. # E.g. running a fuzzer to generate testcases or launcher script. plt = environment.platform() is_android = environment.is_android(plt) runs_on_device = is_android or plt == 'FUCHSIA' if runs_on_device and (not testcase_run or launcher): plt = 'LINUX' # Lower down testcase timeout slightly to account for time for crash analysis. timeout -= CRASH_ANALYSIS_TIME # LeakSanitizer hack - give time for stdout/stderr processing. lsan = environment.get_value('LSAN', False) if lsan: timeout -= LSAN_ANALYSIS_TIME # Initialize variables. adb_output = None process_output = '' process_status = None return_code = 0 process_poll_interval = environment.get_value('PROCESS_POLL_INTERVAL', 0.5) start_time = time.time() watch_for_process_exit = (environment.get_value('WATCH_FOR_PROCESS_EXIT') if is_android else True) window_list = [] # Get gesture start time from last element in gesture list. gestures = copy.deepcopy(gestures) if gestures and gestures[-1].startswith('Trigger'): gesture_start_time = int(gestures[-1].split(':')[1]) gestures.pop() else: gesture_start_time = timeout // 2 if is_android: # Clear the log upfront. android.logger.clear_log() # Run the app. adb_output = android.adb.run_command(cmdline, timeout=timeout) else: cmd, args = shell.get_command_and_arguments(cmdline) process_output = mozprocess.processhandler.StoreOutput() process_status = ProcessStatus() try: process_handle = mozprocess.ProcessHandlerMixin( cmd, args, cwd=current_working_directory, shell=need_shell, processOutputLine=[process_output], onFinish=[process_status], ignore_children=ignore_children) start_process(process_handle) except: logs.log_error('Exception occurred when running command: %s.' % cmdline) return None, None, '' while True: time.sleep(process_poll_interval) # Run the gestures at gesture_start_time or in case we didn't find windows # in the last try. if (gestures and time.time() - start_time >= gesture_start_time and not window_list): # In case, we don't find any windows, we increment the gesture start time # so that the next check is after 1 second. gesture_start_time += 1 if plt == 'LINUX': linux.gestures.run_gestures(gestures, process_handle.pid, process_status, start_time, timeout, window_list) elif plt == 'WINDOWS': windows.gestures.run_gestures(gestures, process_handle.pid, process_status, start_time, timeout, window_list) elif is_android: android.gestures.run_gestures(gestures, start_time, timeout) # TODO(mbarbella): We add a fake window here to prevent gestures on # Android from getting executed more than once. window_list = ['FAKE'] if time.time() - start_time >= timeout: break # Collect the process output. output = (android.logger.log_output() if is_android else b'\n'.join(process_output.output)) output = utils.decode_to_unicode(output) if crash_analyzer.is_memory_tool_crash(output): break # Check if we need to bail out on process exit. if watch_for_process_exit: # If |watch_for_process_exit| is set, then we already completed running # our app launch command. So, we can bail out. if is_android: break # On desktop, we bail out as soon as the process finishes. if process_status and process_status.finished: # Wait for process shutdown and set return code. process_handle.wait(timeout=PROCESS_CLEANUP_WAIT_TIME) break # Process output based on platform. if is_android: # Get current log output. If device is in reboot mode, logcat automatically # waits for device to be online. time.sleep(ANDROID_CRASH_LOGCAT_WAIT_TIME) output = android.logger.log_output() if android.constants.LOW_MEMORY_REGEX.search(output): # If the device is low on memory, we should force reboot and bail out to # prevent device from getting in a frozen state. logs.log('Device is low on memory, rebooting.', output=output) android.adb.hard_reset() android.adb.wait_for_device() elif android.adb.time_since_last_reboot() < time.time() - start_time: # Check if a reboot has happened, if yes, append log output before reboot # and kernel logs content to output. log_before_last_reboot = android.logger.log_output_before_last_reboot( ) kernel_log = android.adb.get_kernel_log_content() output = '%s%s%s%s%s' % ( log_before_last_reboot, utils.get_line_seperator('Device rebooted'), output, utils.get_line_seperator('Kernel Log'), kernel_log) # Make sure to reset SE Linux Permissive Mode. This can be done cheaply # in ~0.15 sec and is needed especially between runs for kernel crashes. android.adb.run_as_root() android.settings.change_se_linux_to_permissive_mode() return_code = 1 # Add output from adb to the front. if adb_output: output = '%s\n\n%s' % (adb_output, output) # Kill the application if it is still running. We do this at the end to # prevent this from adding noise to the logcat output. task_name = environment.get_value('TASK_NAME') child_process_termination_pattern = environment.get_value( 'CHILD_PROCESS_TERMINATION_PATTERN') if task_name == 'fuzz' and child_process_termination_pattern: # In some cases, we do not want to terminate the application after each # run to avoid long startup times (e.g. for chrome). Terminate processes # matching a particular pattern for light cleanup in this case. android.adb.kill_processes_and_children_matching_name( child_process_termination_pattern) else: # There is no special termination behavior. Simply stop the application. android.app.stop() else: # Get the return code in case the process has finished already. # If the process hasn't finished, return_code will be None which is what # callers expect unless the output indicates a crash. return_code = process_handle.poll() # If the process is still running, then terminate it. if not process_status.finished: launcher_with_interpreter = shell.get_execute_command( launcher) if launcher else None if (launcher_with_interpreter and cmdline.startswith(launcher_with_interpreter)): # If this was a launcher script, we KILL all child processes created # except for APP_NAME. # It is expected that, if the launcher script terminated normally, it # cleans up all the child processes it created itself. terminate_root_and_child_processes(process_handle.pid) else: try: # kill() here actually sends SIGTERM on posix. process_handle.kill() except: pass if lsan: time.sleep(LSAN_ANALYSIS_TIME) output = b'\n'.join(process_output.output) output = utils.decode_to_unicode(output) # X Server hack when max client reached. if ('Maximum number of clients reached' in output or 'Unable to get connection to X server' in output): logs.log_error('Unable to connect to X server, exiting.') os.system('sudo killall -9 Xvfb blackbox >/dev/null 2>&1') sys.exit(0) if testcase_run and (crash_analyzer.is_memory_tool_crash(output) or crash_analyzer.is_check_failure_crash(output)): return_code = 1 # If a crash is found, then we add the memory state as well. if return_code and is_android: ps_output = android.adb.get_ps_output() if ps_output: output += utils.get_line_seperator('Memory Statistics') output += ps_output if return_code: logs.log_warn('Process (%s) ended with exit code (%s).' % (repr(cmdline), str(return_code)), output=output) return return_code, round(time.time() - start_time, 1), output
def clear_device_temp_directories(): """Clear device specific temp directories.""" if environment.is_android() and environment.get_value('ANDROID_SERIAL'): from platforms import android android.device.clear_temp_directories()
def get_crash_info(output): """Parse crash output to get (local) minidump path and any other information useful for crash uploading, and store in a CrashReportInfo object.""" crash_stacks_directory = environment.get_value('CRASH_STACKTRACES_DIR') output_lines = output.splitlines() num_lines = len(output_lines) is_android = environment.is_android() for i, line in enumerate(output_lines): if is_android: # If we are on Android, the dump extraction is more complicated. # The location placed in the crash-stacktrace is of the dump itself but # in fact only the MIME of the dump exists, and will have a different # extension. We need to pull the MIME and process it. match = re.match(CRASH_DUMP_PATH_MARKER, line) if not match: continue minidump_mime_filename_base = None for j in range(i + 1, num_lines): line = output_lines[j] match = re.match(r'(.*)\.dmp', line) if match: minidump_mime_filename_base = os.path.basename(match.group(1).strip()) break if not minidump_mime_filename_base: logs.log_error('Minidump marker was found, but no path in stacktrace.') return None # Look for MIME. If none found, bail. # We might not have copied over the crash dumps yet (copying is buffered), # so we want to search both the original directory and the one to which # the minidumps should later be copied. device_directories_to_search = [ constants.CRASH_DUMPS_DIR, os.path.dirname(line.strip()) ] device_minidump_search_paths = [] device_minidump_mime_path = None for device_directory in device_directories_to_search: device_minidump_mime_potential_paths = adb.run_shell_command( ['ls', '"%s"' % device_directory], root=True).splitlines() device_minidump_search_paths += device_minidump_mime_potential_paths for potential_path in device_minidump_mime_potential_paths: # Check that we actually found a file, and the right one (not logcat). if 'No such file or directory' in potential_path: continue if minidump_mime_filename_base not in potential_path: continue if '.up' in potential_path or '.dmp' in potential_path: device_minidump_mime_path = os.path.join(device_directory, potential_path) break # Break if we found a path. if device_minidump_mime_path is not None: break # If we still didn't find a minidump path, bail. if device_minidump_mime_path is None: logs.log_error('Could not get MIME path from ls:\n%s' % str(device_minidump_search_paths)) return None # Pull out MIME and parse to minidump file and MIME parameters. minidump_mime_filename = '%s.mime' % minidump_mime_filename_base local_minidump_mime_path = os.path.join(crash_stacks_directory, minidump_mime_filename) adb.run_command([ 'pull', '"%s"' % device_minidump_mime_path, local_minidump_mime_path ]) if not os.path.exists(local_minidump_mime_path): logs.log_error('Could not pull MIME from %s to %s.' % (device_minidump_mime_path, local_minidump_mime_path)) return None crash_info = parse_mime_to_crash_report_info(local_minidump_mime_path) if crash_info is None: return None crash_info.unsymbolized_stacktrace = output return crash_info # Other platforms are not currently supported. logs.log_error('Unable to fetch crash information for this platform.') return None # Could not find dump location, bail out. This could also happen when we don't # have a minidump location in stack at all, e.g. when testcase does not crash # during minimization. return None
def clear_device_temp_directories(): """Clear device specific temp directories.""" if environment.is_android(): from platforms import android android.device.clear_temp_directories()
def setup_testcase(testcase, job_type, fuzzer_override=None): """Sets up the testcase and needed dependencies like fuzzer, data bundle, etc.""" fuzzer_name = fuzzer_override or testcase.fuzzer_name task_name = environment.get_value('TASK_NAME') testcase_fail_wait = environment.get_value('FAIL_WAIT') testcase_id = testcase.key.id() # Clear testcase directories. shell.clear_testcase_directories() # Adjust the test timeout value if this is coming from an user uploaded # testcase. if testcase.uploader_email: _set_timeout_value_from_user_upload(testcase_id) # Update the fuzzer if necessary in order to get the updated data bundle. if fuzzer_name: try: update_successful = update_fuzzer_and_data_bundles(fuzzer_name) except errors.InvalidFuzzerError: # Close testcase and don't recreate tasks if this fuzzer is invalid. testcase.open = False testcase.fixed = 'NA' testcase.set_metadata('fuzzer_was_deleted', True) logs.log_error('Closed testcase %d with invalid fuzzer %s.' % (testcase_id, fuzzer_name)) error_message = 'Fuzzer %s no longer exists' % fuzzer_name data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) return None, None, None if not update_successful: error_message = 'Unable to setup fuzzer %s' % fuzzer_name data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) tasks.add_task( task_name, testcase_id, job_type, wait_time=testcase_fail_wait) return None, None, None # Extract the testcase and any of its resources to the input directory. file_list, input_directory, testcase_file_path = unpack_testcase(testcase) if not file_list: error_message = 'Unable to setup testcase %s' % testcase_file_path data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, error_message) tasks.add_task( task_name, testcase_id, job_type, wait_time=testcase_fail_wait) return None, None, None # For Android/Fuchsia, we need to sync our local testcases directory with the # one on the device. if environment.is_android(): _copy_testcase_to_device_and_setup_environment(testcase, testcase_file_path) # Push testcases to worker. if environment.is_trusted_host(): from bot.untrusted_runner import file_host file_host.push_testcases_to_worker() # Copy global blacklist into local blacklist. is_lsan_enabled = environment.get_value('LSAN') if is_lsan_enabled: # Get local blacklist without this testcase's entry. leak_blacklist.copy_global_to_local_blacklist(excluded_testcase=testcase) prepare_environment_for_testcase(testcase, job_type, task_name) return file_list, input_directory, testcase_file_path
def get_crash_data(crash_data, symbolize_flag=True, fuzz_target=None, already_symbolized=False, detect_ooms_and_hangs=None): """Get crash parameters from crash data. Crash parameters include crash type, address, state and stacktrace. If the stacktrace is not already symbolized, we will try to symbolize it unless |symbolize| flag is set to False. Symbolized stacktrace will contain inline frames, but we do exclude them for purposes of crash state generation (helps in testcase deduplication).""" # Decide whether to symbolize or not symbolize the input stacktrace. # Note that Fuchsia logs are always symbolized. if symbolize_flag: # Defer imports since stack_symbolizer pulls in a lot of things. from crash_analysis.stack_parsing import stack_symbolizer crash_stacktrace_with_inlines = stack_symbolizer.symbolize_stacktrace( crash_data, enable_inline_frames=True) crash_stacktrace_without_inlines = stack_symbolizer.symbolize_stacktrace( crash_data, enable_inline_frames=False) else: # We are explicitly indicated to not symbolize using |symbolize_flag|. There # is no distinction between inline and non-inline frames for an unsymbolized # stacktrace. crash_stacktrace_with_inlines = crash_data crash_stacktrace_without_inlines = crash_data # Additional stack frame ignore regexes. custom_stack_frame_ignore_regexes = (local_config.ProjectConfig().get( 'stacktrace.stack_frame_ignore_regexes', [])) if environment.get_value('TASK_NAME') == 'analyze': detect_v8_runtime_errors = True else: detect_v8_runtime_errors = environment.get_value( 'DETECT_V8_RUNTIME_ERRORS', False) fuzz_target = fuzz_target or environment.get_value('FUZZ_TARGET') redzone_size = environment.get_value('REDZONE') if detect_ooms_and_hangs is None: detect_ooms_and_hangs = ( environment.get_value('REPORT_OOMS_AND_HANGS') and (not redzone_size or redzone_size <= MAX_REDZONE_SIZE_FOR_OOMS_AND_HANGS)) include_ubsan = 'halt_on_error=0' not in environment.get_value( 'UBSAN_OPTIONS', '') stack_parser = stacktraces.StackParser( symbolized=symbolize_flag or already_symbolized, detect_ooms_and_hangs=detect_ooms_and_hangs, detect_v8_runtime_errors=detect_v8_runtime_errors, custom_stack_frame_ignore_regexes=custom_stack_frame_ignore_regexes, fuzz_target=fuzz_target, include_ubsan=include_ubsan) result = stack_parser.parse(crash_stacktrace_without_inlines) # Use stacktrace with inlines for the result. if result.crash_stacktrace: result.crash_stacktrace = crash_stacktrace_with_inlines # Linkify Android stacktrace. if environment.is_android() and (result.found_android_kernel_crash or result.is_kasan): linkify_android_stacktrace(result) return result