Beispiel #1
0
def start_heartbeat(heartbeat_command):
  """Start the heartbeat (in another process)."""
  global _heartbeat_handle
  if _heartbeat_handle:
    # If heartbeat is already started, no work to do. Bail out.
    return

  try:
    command = shell.get_command(heartbeat_command)
    process_handle = subprocess.Popen(command)
  except Exception:
    logs.log_error(
        'Unable to start heartbeat process (%s).' % heartbeat_command)
    return

  # If heartbeat is successfully started, set its handle now.
  _heartbeat_handle = process_handle

  # Artificial delay to let heartbeat's start time update first.
  sleep(HEARTBEAT_START_WAIT_TIME)
Beispiel #2
0
def start_bot(bot_command):
  """Start the bot process."""
  command = shell.get_command(bot_command)

  # Wait until the process terminates or until run timed out.
  run_timeout = environment.get_value('RUN_TIMEOUT')
  if run_timeout and run_timeout > MAX_SUBPROCESS_TIMEOUT:
    logs.log_error(
        'Capping RUN_TIMEOUT to max allowed value: %d' % MAX_SUBPROCESS_TIMEOUT)
    run_timeout = MAX_SUBPROCESS_TIMEOUT

  try:
    result = subprocess.run(
        command,
        timeout=run_timeout,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        check=False)
    exit_code = result.returncode
    output = result.stdout
  except subprocess.TimeoutExpired as e:
    exit_code = 0
    output = e.stdout
  except Exception:
    logs.log_error('Unable to start bot process (%s).' % bot_command)
    return 1

  if output:
    output = output.decode('utf-8', errors='ignore')
  log_message = f'Command: {command} (exit={exit_code})\n{output}'

  if exit_code == 0:
    logs.log(log_message)
  elif exit_code == 1:
    # Anecdotally, exit=1 means there's a fatal Python exception.
    logs.log_error(log_message)
  else:
    logs.log_warn(log_message)

  return exit_code
Beispiel #3
0
def run_process(cmdline,
                current_working_directory=None,
                timeout=DEFAULT_TEST_TIMEOUT,
                need_shell=False,
                gestures=None,
                env_copy=None,
                testcase_run=True,
                ignore_children=True):
    """Executes a process with a given command line and other parameters."""
    if environment.is_trusted_host() and testcase_run:
        from bot.untrusted_runner import remote_process_host
        return remote_process_host.run_process(cmdline,
                                               current_working_directory,
                                               timeout, need_shell, gestures,
                                               env_copy, testcase_run,
                                               ignore_children)

    if gestures is None:
        gestures = []

    if env_copy:
        os.environ.update(env_copy)

    # FIXME(mbarbella): Using LAUNCHER_PATH here is error prone. It forces us to
    # do certain operations before fuzzer setup (e.g. bad build check).
    launcher = environment.get_value('LAUNCHER_PATH')

    # This is used when running scripts on native linux OS and not on the device.
    # E.g. running a fuzzer to generate testcases or launcher script.
    plt = environment.platform()
    runs_on_device = environment.is_android(plt) or plt == 'FUCHSIA'
    if runs_on_device and (not testcase_run or launcher):
        plt = 'LINUX'

    is_android = environment.is_android(plt)

    # Lower down testcase timeout slightly to account for time for crash analysis.
    timeout -= CRASH_ANALYSIS_TIME

    # LeakSanitizer hack - give time for stdout/stderr processing.
    lsan = environment.get_value('LSAN', False)
    if lsan:
        timeout -= LSAN_ANALYSIS_TIME

    # Initialize variables.
    adb_output = None
    process_output = ''
    process_status = None
    return_code = 0
    process_poll_interval = environment.get_value('PROCESS_POLL_INTERVAL', 0.5)
    start_time = time.time()
    watch_for_process_exit = (environment.get_value('WATCH_FOR_PROCESS_EXIT')
                              if is_android else True)
    window_list = []

    # Get gesture start time from last element in gesture list.
    gestures = copy.deepcopy(gestures)
    if gestures and gestures[-1].startswith('Trigger'):
        gesture_start_time = int(gestures[-1].split(':')[1])
        gestures.pop()
    else:
        gesture_start_time = timeout // 2

    if is_android:
        # Clear the log upfront.
        android.logger.clear_log()

        # Run the app.
        adb_output = android.adb.run_command(cmdline, timeout=timeout)
    else:
        cmd = shell.get_command(cmdline)

        process_output = mozprocess.processhandler.StoreOutput()
        process_status = ProcessStatus()
        try:
            process_handle = mozprocess.ProcessHandlerMixin(
                cmd,
                args=None,
                cwd=current_working_directory,
                shell=need_shell,
                processOutputLine=[process_output],
                onFinish=[process_status],
                ignore_children=ignore_children)
            start_process(process_handle)
        except:
            logs.log_error('Exception occurred when running command: %s.' %
                           cmdline)
            return None, None, ''

    while True:
        time.sleep(process_poll_interval)

        # Run the gestures at gesture_start_time or in case we didn't find windows
        # in the last try.
        if (gestures and time.time() - start_time >= gesture_start_time
                and not window_list):
            # In case, we don't find any windows, we increment the gesture start time
            # so that the next check is after 1 second.
            gesture_start_time += 1

            if plt == 'LINUX':
                linux.gestures.run_gestures(gestures, process_handle.pid,
                                            process_status, start_time,
                                            timeout, window_list)
            elif plt == 'WINDOWS':
                windows.gestures.run_gestures(gestures, process_handle.pid,
                                              process_status, start_time,
                                              timeout, window_list)
            elif is_android:
                android.gestures.run_gestures(gestures, start_time, timeout)

                # TODO(mbarbella): We add a fake window here to prevent gestures on
                # Android from getting executed more than once.
                window_list = ['FAKE']

        if time.time() - start_time >= timeout:
            break

        # Collect the process output.
        output = (android.logger.log_output()
                  if is_android else b'\n'.join(process_output.output))
        output = utils.decode_to_unicode(output)
        if crash_analyzer.is_memory_tool_crash(output):
            break

        # Check if we need to bail out on process exit.
        if watch_for_process_exit:
            # If |watch_for_process_exit| is set, then we already completed running
            # our app launch command. So, we can bail out.
            if is_android:
                break

            # On desktop, we bail out as soon as the process finishes.
            if process_status and process_status.finished:
                # Wait for process shutdown and set return code.
                process_handle.wait(timeout=PROCESS_CLEANUP_WAIT_TIME)
                break

    # Process output based on platform.
    if is_android:
        # Get current log output. If device is in reboot mode, logcat automatically
        # waits for device to be online.
        time.sleep(ANDROID_CRASH_LOGCAT_WAIT_TIME)
        output = android.logger.log_output()

        if android.constants.LOW_MEMORY_REGEX.search(output):
            # If the device is low on memory, we should force reboot and bail out to
            # prevent device from getting in a frozen state.
            logs.log('Device is low on memory, rebooting.', output=output)
            android.adb.hard_reset()
            android.adb.wait_for_device()

        elif android.adb.time_since_last_reboot() < time.time() - start_time:
            # Check if a reboot has happened, if yes, append log output before reboot
            # and kernel logs content to output.
            log_before_last_reboot = android.logger.log_output_before_last_reboot(
            )
            kernel_log = android.adb.get_kernel_log_content()
            output = '%s%s%s%s%s' % (
                log_before_last_reboot,
                utils.get_line_seperator('Device rebooted'), output,
                utils.get_line_seperator('Kernel Log'), kernel_log)
            # Make sure to reset SE Linux Permissive Mode. This can be done cheaply
            # in ~0.15 sec and is needed especially between runs for kernel crashes.
            android.adb.run_as_root()
            android.settings.change_se_linux_to_permissive_mode()
            return_code = 1

        # Add output from adb to the front.
        if adb_output:
            output = '%s\n\n%s' % (adb_output, output)

        # Kill the application if it is still running. We do this at the end to
        # prevent this from adding noise to the logcat output.
        task_name = environment.get_value('TASK_NAME')
        child_process_termination_pattern = environment.get_value(
            'CHILD_PROCESS_TERMINATION_PATTERN')
        if task_name == 'fuzz' and child_process_termination_pattern:
            # In some cases, we do not want to terminate the application after each
            # run to avoid long startup times (e.g. for chrome). Terminate processes
            # matching a particular pattern for light cleanup in this case.
            android.adb.kill_processes_and_children_matching_name(
                child_process_termination_pattern)
        else:
            # There is no special termination behavior. Simply stop the application.
            android.app.stop()

    else:
        # Get the return code in case the process has finished already.
        # If the process hasn't finished, return_code will be None which is what
        # callers expect unless the output indicates a crash.
        return_code = process_handle.poll()

        # If the process is still running, then terminate it.
        if not process_status.finished:
            launcher_with_interpreter = shell.get_execute_command(
                launcher) if launcher else None
            if (launcher_with_interpreter
                    and cmdline.startswith(launcher_with_interpreter)):
                # If this was a launcher script, we KILL all child processes created
                # except for APP_NAME.
                # It is expected that, if the launcher script terminated normally, it
                # cleans up all the child processes it created itself.
                terminate_root_and_child_processes(process_handle.pid)
            else:
                try:
                    # kill() here actually sends SIGTERM on posix.
                    process_handle.kill()
                except:
                    pass

        if lsan:
            time.sleep(LSAN_ANALYSIS_TIME)

        output = b'\n'.join(process_output.output)
        output = utils.decode_to_unicode(output)

        # X Server hack when max client reached.
        if ('Maximum number of clients reached' in output
                or 'Unable to get connection to X server' in output):
            logs.log_error('Unable to connect to X server, exiting.')
            os.system('sudo killall -9 Xvfb blackbox >/dev/null 2>&1')
            sys.exit(0)

    if testcase_run and (crash_analyzer.is_memory_tool_crash(output)
                         or crash_analyzer.is_check_failure_crash(output)):
        return_code = 1

    # If a crash is found, then we add the memory state as well.
    if return_code and is_android:
        ps_output = android.adb.get_ps_output()
        if ps_output:
            output += utils.get_line_seperator('Memory Statistics')
            output += ps_output

    if return_code:
        logs.log_warn('Process (%s) ended with exit code (%s).' %
                      (repr(cmdline), str(return_code)),
                      output=output)

    return return_code, round(time.time() - start_time, 1), output