Exemplo n.º 1
0
    def run_test_setup(self, test):
        self.log.info("starting raptor test: %s" % test['name'])
        self.log.info("test settings: %s" % str(test))
        self.log.info("raptor config: %s" % str(self.config))

        if test.get('type') == "benchmark":
            self.serve_benchmark_source(test)

        gen_test_config(
            self.config['app'],
            test['name'],
            self.control_server.port,
            self.post_startup_delay,
            host=self.config['host'],
            b_port=self.benchmark_port,
            debug_mode=1 if self.debug_mode else 0,
            browser_cycle=test.get('browser_cycle', 1),
        )

        self.install_raptor_webext()

        if test.get("preferences") is not None:
            self.set_browser_test_prefs(test['preferences'])

        # if 'alert_on' was provided in the test INI, add to our config for results/output
        self.config['subtest_alert_on'] = test.get('alert_on')
Exemplo n.º 2
0
    def run_test(self, test, timeout=None):
        self.log.info("starting raptor test: %s" % test['name'])
        self.log.info("test settings: %s" % str(test))
        self.log.info("raptor config: %s" % str(self.config))

        # benchmark-type tests require the benchmark test to be served out
        if test.get('type') == "benchmark":
            self.benchmark = Benchmark(self.config, test)
            benchmark_port = int(self.benchmark.port)
        else:
            benchmark_port = 0

        gen_test_config(self.config['app'], test['name'],
                        self.control_server.port, benchmark_port)

        # must intall raptor addon each time because we dynamically update some content
        raptor_webext = os.path.join(webext_dir, 'raptor')
        self.log.info("installing webext %s" % raptor_webext)
        self.profile.addons.install(raptor_webext)
        # on firefox we can get an addon id; chrome addon actually is just cmd line arg
        if self.config['app'] == "firefox":
            webext_id = self.profile.addons.addon_details(raptor_webext)['id']

        # some tests require tools to playback the test pages
        if test.get('playback', None) is not None:
            self.get_playback_config(test)
            # startup the playback tool
            self.playback = get_playback(self.config)

        self.runner.start()

        proc = self.runner.process_handler
        self.output_handler.proc = proc
        self.control_server.browser_proc = proc

        try:
            self.runner.wait(timeout)
        finally:
            try:
                self.runner.check_for_crashes()
            except NotImplementedError:  # not implemented for Chrome
                pass

        if self.playback is not None:
            self.playback.stop()

        # remove the raptor webext; as it must be reloaded with each subtest anyway
        # applies to firefox only; chrome the addon is actually just cmd line arg
        if self.config['app'] == "firefox":
            self.log.info("removing webext %s" % raptor_webext)
            self.profile.addons.remove_addon(webext_id)

        if self.runner.is_running():
            self.log("Application timed out after {} seconds".format(timeout))
            self.runner.stop()
Exemplo n.º 3
0
    def run_test(self, test, timeout=None):
        self.log.info("starting raptor test: %s" % test['name'])
        gen_test_config(self.config['app'], test['name'],
                        self.control_server.port)

        # must intall raptor addon each time because we dynamically update some content
        raptor_webext = os.path.join(webext_dir, 'raptor')
        self.log.info("installing webext %s" % raptor_webext)
        self.profile.addons.install(raptor_webext)
        webext_id = self.profile.addons.addon_details(raptor_webext)['id']

        # some tests require tools to playback the test pages
        if test.get('playback', None) is not None:
            self.get_playback_config(test)
            # startup the playback tool
            self.playback = get_playback(self.config)

        self.runner.start()

        proc = self.runner.process_handler
        self.output_handler.proc = proc
        self.control_server.browser_proc = proc

        try:
            self.runner.wait(timeout)
        finally:
            try:
                self.runner.check_for_crashes()
            except NotImplementedError:  # not implemented for Chrome
                pass

        if self.playback is not None:
            self.playback.stop()

        # remove the raptor webext; as it must be reloaded with each subtest anyway
        self.log.info("removing webext %s" % raptor_webext)
        self.profile.addons.remove_addon(webext_id)

        if self.runner.is_running():
            self.log("Application timed out after {} seconds".format(timeout))
            self.runner.stop()
Exemplo n.º 4
0
    def run_test_setup(self, test):
        super(WebExtension, self).run_test_setup(test)

        LOG.info("starting web extension test: %s" % test["name"])
        LOG.info("test settings: %s" % str(test))
        LOG.info("web extension config: %s" % str(self.config))

        if test.get("type") == "benchmark":
            self.serve_benchmark_source(test)

        gen_test_config(
            test["name"],
            self.control_server.port,
            self.post_startup_delay,
            host=self.config["host"],
            b_port=int(self.benchmark.port) if self.benchmark else 0,
            debug_mode=1 if self.debug_mode else 0,
            browser_cycle=test.get("browser_cycle", 1),
        )

        self.install_raptor_webext()
Exemplo n.º 5
0
    def run_test(self, test, timeout=None):
        self.log.info("starting raptor test: %s" % test['name'])
        gen_test_config(self.config['app'], test['name'],
                        self.control_server.port)

        self.profile.addons.install(os.path.join(webext_dir, 'raptor'))

        # some tests require tools to playback the test pages
        if test.get('playback', None) is not None:
            self.get_playback_config(test)
            # startup the playback tool
            self.playback = get_playback(self.config)

        self.runner.start()

        first_time = int(time.time()) * 1000
        proc = self.runner.process_handler
        self.output_handler.proc = proc

        try:
            self.runner.wait(timeout)
        finally:
            try:
                self.runner.check_for_crashes()
            except NotImplementedError:  # not implemented for Chrome
                pass

        if self.playback is not None:
            self.playback.stop()

        if self.runner.is_running():
            self.log("Application timed out after {} seconds".format(timeout))
            self.runner.stop()

        proc.output.append(
            "__startBeforeLaunchTimestamp%d__endBeforeLaunchTimestamp" %
            first_time)
        proc.output.append(
            "__startAfterTerminationTimestamp%d__endAfterTerminationTimestamp"
            % (int(time.time()) * 1000))
Exemplo n.º 6
0
    def run_test(self, test, timeout=None):
        self.log.info("starting raptor test: %s" % test['name'])
        self.log.info("test settings: %s" % str(test))
        self.log.info("raptor config: %s" % str(self.config))

        # benchmark-type tests require the benchmark test to be served out
        if test.get('type') == "benchmark":
            self.benchmark = Benchmark(self.config, test)
            benchmark_port = int(self.benchmark.port)
        else:
            benchmark_port = 0

        gen_test_config(self.config['app'], test['name'],
                        self.control_server.port, benchmark_port)

        # for android we must make the benchmarks server available to the device
        if self.config['app'] == "geckoview":
            self.log.info(
                "making the raptor benchmarks server port available to device")
            _tcp_port = "tcp:%s" % benchmark_port
            self.device.create_socket_connection('reverse', _tcp_port,
                                                 _tcp_port)

        # must intall raptor addon each time because we dynamically update some content
        raptor_webext = os.path.join(webext_dir, 'raptor')
        self.log.info("installing webext %s" % raptor_webext)
        self.profile.addons.install(raptor_webext)

        # add test specific preferences
        if test.get("preferences", None) is not None:
            if self.config['app'] == "firefox":
                self.profile.set_preferences(json.loads(test['preferences']))
            else:
                self.log.info("preferences were configured for the test, \
                              but we do not install them on non Firefox browsers."
                              )

        # on firefox we can get an addon id; chrome addon actually is just cmd line arg
        if self.config['app'] in ["firefox", "geckoview"]:
            webext_id = self.profile.addons.addon_details(raptor_webext)['id']

        # some tests require tools to playback the test pages
        if test.get('playback', None) is not None:
            self.get_playback_config(test)
            # startup the playback tool
            self.playback = get_playback(self.config)

        # for geckoview we must copy the profile onto the device and set perms
        if self.config['app'] == "geckoview":
            if not self.device.is_app_installed(self.config['binary']):
                raise Exception('%s is not installed' % self.config['binary'])

            self.log.info("copying firefox profile onto the android device")
            self.device_profile = "/sdcard/raptor-profile"
            if self.device.is_dir(self.device_profile):
                self.device.rm(self.device_profile, recursive=True)

            self.device.mkdir(self.device_profile)
            self.device.push(self.profile.profile, self.device_profile)

            self.log.info("setting permisions to profile dir on the device")
            self.device.chmod(self.device_profile, recursive=True)

            # now start the geckoview app
            self.log.info("starting %s" % self.config['app'])

            extra_args = [
                "-profile", self.device_profile, "--es", "env0",
                "LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6"
            ]

            try:
                # make sure the geckoview app is not running before
                # attempting to start.
                self.device.stop_application(self.config['binary'])
                self.device.launch_activity(self.config['binary'],
                                            "GeckoViewActivity",
                                            extra_args=extra_args,
                                            url='about:blank',
                                            fail_if_running=False)
            except Exception:
                self.log.error("Exception launching %s" %
                               self.config['binary'])
                raise
            self.control_server.device = self.device
            self.control_server.app_name = self.config['binary']

        else:
            # now start the desktop browser
            self.log.info("starting %s" % self.config['app'])

            self.runner.start()
            proc = self.runner.process_handler
            self.output_handler.proc = proc

            self.control_server.browser_proc = proc

        # set our cs flag to indicate we are running the browser/app
        self.control_server._finished = False

        # convert to seconds and account for page cycles
        timeout = int(timeout / 1000) * int(test['page_cycles'])
        try:
            elapsed_time = 0
            while not self.control_server._finished:
                time.sleep(1)
                elapsed_time += 1
                if elapsed_time > (timeout) - 5:  # stop 5 seconds early
                    self.log.info(
                        "application timed out after {} seconds".format(
                            timeout))
                    self.control_server.wait_for_quit()
                    break
        finally:
            if self.config['app'] != "geckoview":
                try:
                    self.runner.check_for_crashes()
                except NotImplementedError:  # not implemented for Chrome
                    pass
            # TODO: if on geckoview is there some cleanup here i.e. check for crashes?

        if self.playback is not None:
            self.playback.stop()

        # remove the raptor webext; as it must be reloaded with each subtest anyway
        # applies to firefox only; chrome the addon is actually just cmd line arg
        if self.config['app'] in ["firefox", "geckoview"]:
            self.log.info("removing webext %s" % raptor_webext)
            self.profile.addons.remove_addon(webext_id)

        if self.config['app'] != "geckoview":
            if self.runner.is_running():
                self.runner.stop()
Exemplo n.º 7
0
    def run_test(self, test, timeout=None):
        self.log.info("starting raptor test: %s" % test['name'])
        self.log.info("test settings: %s" % str(test))
        self.log.info("raptor config: %s" % str(self.config))

        # benchmark-type tests require the benchmark test to be served out
        if test.get('type') == "benchmark":
            self.benchmark = Benchmark(self.config, test)
            benchmark_port = int(self.benchmark.port)

            # for android we must make the benchmarks server available to the device
            if self.config['app'] == "geckoview" and self.config['host'] \
                    in ('localhost', '127.0.0.1'):
                self.log.info(
                    "making the raptor benchmarks server port available to device"
                )
                _tcp_port = "tcp:%s" % benchmark_port
                self.device.create_socket_connection('reverse', _tcp_port,
                                                     _tcp_port)
        else:
            benchmark_port = 0

        gen_test_config(self.config['app'],
                        test['name'],
                        self.control_server.port,
                        self.post_startup_delay,
                        host=self.config['host'],
                        b_port=benchmark_port,
                        debug_mode=1 if self.debug_mode else 0)

        # must intall raptor addon each time because we dynamically update some content
        # note: for chrome the addon is just a list of paths that ultimately are added
        # to the chromium command line '--load-extension' argument
        raptor_webext = os.path.join(webext_dir, 'raptor')
        self.log.info("installing webext %s" % raptor_webext)
        self.profile.addons.install(raptor_webext)

        # add test specific preferences
        if test.get("preferences", None) is not None:
            if self.config['app'] == "firefox":
                self.profile.set_preferences(json.loads(test['preferences']))
            else:
                self.log.info("preferences were configured for the test, \
                              but we do not install them on non Firefox browsers."
                              )

        # on firefox we can get an addon id; chrome addon actually is just cmd line arg
        if self.config['app'] in ["firefox", "geckoview"]:
            webext_id = self.profile.addons.addon_details(raptor_webext)['id']

        # for android/geckoview, create a top-level raptor folder on the device
        # sdcard; if it already exists remove it so we start fresh each time
        if self.config['app'] == "geckoview":
            self.device_raptor_dir = "/sdcard/raptor"
            self.config['device_raptor_dir'] = self.device_raptor_dir
            if self.device.is_dir(self.device_raptor_dir):
                self.log.info("deleting existing device raptor dir: %s" %
                              self.device_raptor_dir)
                self.device.rm(self.device_raptor_dir, recursive=True)
            self.log.info("creating raptor folder on sdcard: %s" %
                          self.device_raptor_dir)
            self.device.mkdir(self.device_raptor_dir)
            self.device.chmod(self.device_raptor_dir, recursive=True)

        # some tests require tools to playback the test pages
        if test.get('playback', None) is not None:
            self.get_playback_config(test)
            # startup the playback tool
            self.playback = get_playback(self.config, self.device)

            # for android we must make the playback server available to the device
            if self.config['app'] == "geckoview" and self.config['host'] \
                    in ('localhost', '127.0.0.1'):
                self.log.info(
                    "making the raptor playback server port available to device"
                )
                _tcp_port = "tcp:8080"
                self.device.create_socket_connection('reverse', _tcp_port,
                                                     _tcp_port)

        if self.config['app'] in ("geckoview", "firefox") and \
           self.config['host'] not in ('localhost', '127.0.0.1'):
            # Must delete the proxy settings from the profile if running
            # the test with a host different from localhost.
            userjspath = os.path.join(self.profile.profile, 'user.js')
            with open(userjspath) as userjsfile:
                prefs = userjsfile.readlines()
            prefs = [pref for pref in prefs if 'network.proxy' not in pref]
            with open(userjspath, 'w') as userjsfile:
                userjsfile.writelines(prefs)

        # for geckoview/android pageload playback we can't use a policy to turn on the
        # proxy; we need to set prefs instead; note that the 'host' may be different
        # than '127.0.0.1' so we must set the prefs accordingly
        if self.config['app'] == "geckoview" and test.get('playback',
                                                          None) is not None:
            self.log.info(
                "setting profile prefs to turn on the geckoview browser proxy")
            no_proxies_on = "localhost, 127.0.0.1, %s" % self.config['host']
            proxy_prefs = {}
            proxy_prefs["network.proxy.type"] = 1
            proxy_prefs["network.proxy.http"] = self.config['host']
            proxy_prefs["network.proxy.http_port"] = 8080
            proxy_prefs["network.proxy.ssl"] = self.config['host']
            proxy_prefs["network.proxy.ssl_port"] = 8080
            proxy_prefs["network.proxy.no_proxies_on"] = no_proxies_on
            self.profile.set_preferences(proxy_prefs)

        # now some final settings, and then startup of the browser under test
        if self.config['app'] == "geckoview":
            # for android/geckoview we must copy the profile onto the device and set perms
            if not self.device.is_app_installed(self.config['binary']):
                raise Exception('%s is not installed' % self.config['binary'])
            self.device_profile = os.path.join(self.device_raptor_dir,
                                               "profile")
            if self.device.is_dir(self.device_profile):
                self.log.info("deleting existing device profile folder: %s" %
                              self.device_profile)
                self.device.rm(self.device_profile, recursive=True)
            self.log.info("creating profile folder on device: %s" %
                          self.device_profile)
            self.device.mkdir(self.device_profile)
            self.log.info("copying firefox profile onto the device")
            self.log.info("note: the profile folder being copied is: %s" %
                          self.profile.profile)
            self.log.info(
                'the adb push cmd copies that profile dir to a new temp dir before copy'
            )
            self.device.push(self.profile.profile, self.device_profile)
            self.device.chmod(self.device_profile, recursive=True)

            # now start the geckoview app
            self.log.info("starting %s" % self.config['app'])

            extra_args = [
                "-profile", self.device_profile, "--es", "env0",
                "LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6"
            ]

            try:
                # make sure the geckoview app is not running before
                # attempting to start.
                self.device.stop_application(self.config['binary'])
                self.device.launch_activity(self.config['binary'],
                                            "GeckoViewActivity",
                                            extra_args=extra_args,
                                            url='about:blank',
                                            e10s=True,
                                            fail_if_running=False)
            except Exception:
                self.log.error("Exception launching %s" %
                               self.config['binary'])
                raise
            self.control_server.device = self.device
            self.control_server.app_name = self.config['binary']

        else:
            # For Firefox we need to set
            # MOZ_DISABLE_NONLOCAL_CONNECTIONS=1 env var before
            # startup when testing release builds from mozilla-beta or
            # mozilla-release. This is because of restrictions on
            # release builds that require webextensions to be signed
            # unless MOZ_DISABLE_NONLOCAL_CONNECTIONS is set to '1'.
            if self.config['app'] == "firefox" and self.config[
                    'is_release_build']:
                self.log.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=1")
                os.environ['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = "1"

            # if running debug-mode, tell Firefox to open the browser console on startup
            # for google chrome, open the devtools on the raptor test tab
            if self.debug_mode:
                if self.config['app'] == "firefox":
                    self.runner.cmdargs.extend(['-jsconsole'])
                if self.config['app'] == "chrome":
                    self.runner.cmdargs.extend(
                        ['--auto-open-devtools-for-tabs'])

            # now start the desktop browser
            self.log.info("starting %s" % self.config['app'])

            # if running a pageload test on google chrome, add the cmd line options
            # to turn on the proxy and ignore security certificate errors
            # if using host localhost, 127.0.0.1.
            if self.config['app'] == "chrome" and test.get('playback',
                                                           None) is not None:
                chrome_args = [
                    '--proxy-server="http=127.0.0.1:8080;' +
                    'https=127.0.0.1:8080;ssl=127.0.0.1:8080"',
                    '--ignore-certificate-errors'
                ]
                if self.config['host'] not in ('localhost', '127.0.0.1'):
                    chrome_args[0] = chrome_args[0].replace(
                        '127.0.0.1', self.config['host'])
                if ' '.join(chrome_args) not in ' '.join(self.runner.cmdargs):
                    self.runner.cmdargs.extend(chrome_args)

            self.runner.start()
            proc = self.runner.process_handler
            self.output_handler.proc = proc

            self.control_server.browser_proc = proc

            # pageload tests need to be able to access non-local connections via mitmproxy
            if self.config['app'] == "firefox" and self.config['is_release_build'] and \
               test.get('playback', None) is not None:
                self.log.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=0")
                os.environ['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = "0"

            # if geckoProfile is enabled, initialize it
            if self.config['gecko_profile'] is True:
                self._init_gecko_profiling(test)
                # tell the control server the gecko_profile dir; the control server will
                # receive the actual gecko profiles from the web ext and will write them
                # to disk; then profiles are picked up by gecko_profile.symbolicate
                self.control_server.gecko_profile_dir = self.gecko_profiler.gecko_profile_dir

        # set our cs flag to indicate we are running the browser/app
        self.control_server._finished = False

        # convert to seconds and account for page cycles
        timeout = int(timeout / 1000) * int(test['page_cycles'])
        # account for the pause the raptor webext runner takes after browser startup
        timeout += int(self.post_startup_delay / 1000)

        # if geckoProfile enabled, give browser more time for profiling
        if self.config['gecko_profile'] is True:
            timeout += 5 * 60

        try:
            elapsed_time = 0
            while not self.control_server._finished:
                time.sleep(1)
                # we only want to force browser-shutdown on timeout if not in debug mode;
                # in debug-mode we leave the browser running (require manual shutdown)
                if not self.debug_mode:
                    elapsed_time += 1
                    if elapsed_time > (timeout) - 5:  # stop 5 seconds early
                        self.log.info(
                            "application timed out after {} seconds".format(
                                timeout))
                        self.control_server.wait_for_quit()
                        break
        finally:
            if self.config['app'] != "geckoview":
                try:
                    self.runner.check_for_crashes()
                except NotImplementedError:  # not implemented for Chrome
                    pass
            # TODO: if on geckoview is there some cleanup here i.e. check for crashes?

        if self.playback is not None:
            self.playback.stop()

        # remove the raptor webext; as it must be reloaded with each subtest anyway
        self.log.info("removing webext %s" % raptor_webext)
        if self.config['app'] in ["firefox", "geckoview"]:
            self.profile.addons.remove_addon(webext_id)

        # for chrome the addon is just a list (appended to cmd line)
        if self.config['app'] in ["chrome", "chrome-android"]:
            self.profile.addons.remove(raptor_webext)

        # gecko profiling symbolication
        if self.config['gecko_profile'] is True:
            self.gecko_profiler.symbolicate()
            # clean up the temp gecko profiling folders
            self.log.info("cleaning up after gecko profiling")
            self.gecko_profiler.clean()

        # browser should be closed by now but this is a backup-shutdown (if not in debug-mode)
        if not self.debug_mode:
            if self.config['app'] != "geckoview":
                if self.runner.is_running():
                    self.runner.stop()
            # TODO the geckoview app should have been shutdown by this point by the
            # control server, but we can double-check here to make sure
        else:
            # in debug mode, and running locally, leave the browser running
            if self.config['run_local']:
                self.log.info(
                    "* debug-mode enabled - please shutdown the browser manually..."
                )
                self.runner.wait(timeout=None)
Exemplo n.º 8
0
    def run_test(self, test, timeout=None):
        self.log.info("starting raptor test: %s" % test['name'])
        self.log.info("test settings: %s" % str(test))
        self.log.info("raptor config: %s" % str(self.config))

        # benchmark-type tests require the benchmark test to be served out
        if test.get('type') == "benchmark":
            self.benchmark = Benchmark(self.config, test)
            benchmark_port = int(self.benchmark.port)
        else:
            benchmark_port = 0

        gen_test_config(self.config['app'], test['name'],
                        self.control_server.port, self.post_startup_delay,
                        benchmark_port)

        # for android we must make the benchmarks server available to the device
        if self.config['app'] == "geckoview":
            self.log.info(
                "making the raptor benchmarks server port available to device")
            _tcp_port = "tcp:%s" % benchmark_port
            self.device.create_socket_connection('reverse', _tcp_port,
                                                 _tcp_port)

        # must intall raptor addon each time because we dynamically update some content
        # note: for chrome the addon is just a list of paths that ultimately are added
        # to the chromium command line '--load-extension' argument
        raptor_webext = os.path.join(webext_dir, 'raptor')
        self.log.info("installing webext %s" % raptor_webext)
        self.profile.addons.install(raptor_webext)

        # add test specific preferences
        if test.get("preferences", None) is not None:
            if self.config['app'] == "firefox":
                self.profile.set_preferences(json.loads(test['preferences']))
            else:
                self.log.info("preferences were configured for the test, \
                              but we do not install them on non Firefox browsers."
                              )

        # on firefox we can get an addon id; chrome addon actually is just cmd line arg
        if self.config['app'] in ["firefox", "geckoview"]:
            webext_id = self.profile.addons.addon_details(raptor_webext)['id']

        # some tests require tools to playback the test pages
        if test.get('playback', None) is not None:
            self.get_playback_config(test)
            # startup the playback tool
            self.playback = get_playback(self.config)

        # for geckoview we must copy the profile onto the device and set perms
        if self.config['app'] == "geckoview":
            if not self.device.is_app_installed(self.config['binary']):
                raise Exception('%s is not installed' % self.config['binary'])

            self.log.info("copying firefox profile onto the android device")
            self.device_profile = "/sdcard/raptor-profile"
            if self.device.is_dir(self.device_profile):
                self.device.rm(self.device_profile, recursive=True)

            self.device.mkdir(self.device_profile)
            self.device.push(self.profile.profile, self.device_profile)

            self.log.info("setting permisions to profile dir on the device")
            self.device.chmod(self.device_profile, recursive=True)

            # now start the geckoview app
            self.log.info("starting %s" % self.config['app'])

            extra_args = [
                "-profile", self.device_profile, "--es", "env0",
                "LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6"
            ]

            try:
                # make sure the geckoview app is not running before
                # attempting to start.
                self.device.stop_application(self.config['binary'])
                self.device.launch_activity(self.config['binary'],
                                            "GeckoViewActivity",
                                            extra_args=extra_args,
                                            url='about:blank',
                                            fail_if_running=False)
            except Exception:
                self.log.error("Exception launching %s" %
                               self.config['binary'])
                raise
            self.control_server.device = self.device
            self.control_server.app_name = self.config['binary']

        else:
            # now start the desktop browser
            self.log.info("starting %s" % self.config['app'])

            # if running a pageload test on google chrome, add the cmd line options
            # to turn on the proxy and ignore security certificate errors
            if self.config['app'] == "chrome" and test.get('playback',
                                                           None) is not None:
                chrome_args = [
                    '--proxy-server="http=127.0.0.1:8080;https=127.0.0.1:8080;ssl=127.0.0.1:8080"',
                    '--ignore-certificate-errors'
                ]
                self.runner.cmdargs.extend(chrome_args)

            self.runner.start()
            proc = self.runner.process_handler
            self.output_handler.proc = proc

            self.control_server.browser_proc = proc

            # if geckoProfile is enabled, initialize it
            if self.config['gecko_profile'] is True:
                self._init_gecko_profiling(test)
                # tell the control server the gecko_profile dir; the control server will
                # receive the actual gecko profiles from the web ext and will write them
                # to disk; then profiles are picked up by gecko_profile.symbolicate
                self.control_server.gecko_profile_dir = self.gecko_profiler.gecko_profile_dir

        # set our cs flag to indicate we are running the browser/app
        self.control_server._finished = False

        # convert to seconds and account for page cycles
        timeout = int(timeout / 1000) * int(test['page_cycles'])
        # account for the pause the raptor webext runner takes after browser startup
        timeout += int(self.post_startup_delay / 1000)

        # if geckoProfile enabled, give browser more time for profiling
        if self.config['gecko_profile'] is True:
            timeout += 5 * 60

        try:
            elapsed_time = 0
            while not self.control_server._finished:
                time.sleep(1)
                elapsed_time += 1
                if elapsed_time > (timeout) - 5:  # stop 5 seconds early
                    self.log.info(
                        "application timed out after {} seconds".format(
                            timeout))
                    self.control_server.wait_for_quit()
                    break
        finally:
            if self.config['app'] != "geckoview":
                try:
                    self.runner.check_for_crashes()
                except NotImplementedError:  # not implemented for Chrome
                    pass
            # TODO: if on geckoview is there some cleanup here i.e. check for crashes?

        if self.playback is not None:
            self.playback.stop()

        # remove the raptor webext; as it must be reloaded with each subtest anyway
        self.log.info("removing webext %s" % raptor_webext)
        if self.config['app'] in ["firefox", "geckoview"]:
            self.profile.addons.remove_addon(webext_id)

        # for chrome the addon is just a list (appended to cmd line)
        if self.config['app'] in ["chrome", "chrome-android"]:
            self.profile.addons.remove(raptor_webext)

        # gecko profiling symbolication
        if self.config['gecko_profile'] is True:
            self.gecko_profiler.symbolicate()
            # clean up the temp gecko profiling folders
            self.log.info("cleaning up after gecko profiling")
            self.gecko_profiler.clean()

        if self.config['app'] != "geckoview":
            if self.runner.is_running():
                self.runner.stop()