Example #1
0
class Raptor(object):
    """Container class for Raptor"""
    def __init__(self, app, binary, run_local=False, obj_path=None):
        self.config = {}
        self.config['app'] = app
        self.config['binary'] = binary
        self.config['platform'] = mozinfo.os
        self.config['processor'] = mozinfo.processor
        self.config['run_local'] = run_local
        self.config['obj_path'] = obj_path
        self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
        self.log = get_default_logger(component='raptor-main')
        self.control_server = None
        self.playback = None
        self.benchmark = None

        # Create the profile; for geckoview we want a firefox profile type
        if self.config['app'] == 'geckoview':
            self.profile = create_profile('firefox')
        else:
            self.profile = create_profile(self.config['app'])

        # Merge in base profiles
        with open(os.path.join(self.profile_data_dir, 'profiles.json'),
                  'r') as fh:
            base_profiles = json.load(fh)['raptor']

        for name in base_profiles:
            path = os.path.join(self.profile_data_dir, name)
            self.log.info("Merging profile: {}".format(path))
            self.profile.merge(path)

        # create results holder
        self.results_handler = RaptorResultsHandler()

        # when testing desktop browsers we use mozrunner to start the browser; when
        # testing on android (i.e. geckoview) we use mozdevice to control the device app

        if self.config['app'] == "geckoview":
            # create the android device handler; it gets initiated and sets up adb etc
            self.log.info("creating android device handler using mozdevice")
            self.device = ADBAndroid(verbose=True)
            self.device.clear_logcat()
        else:
            # create the desktop browser runner
            self.log.info("creating browser runner using mozrunner")
            self.output_handler = OutputHandler()
            process_args = {
                'processOutputLine': [self.output_handler],
            }
            runner_cls = runners[app]
            self.runner = runner_cls(binary,
                                     profile=self.profile,
                                     process_args=process_args)

        self.log.info("raptor config: %s" % str(self.config))

    @property
    def profile_data_dir(self):
        if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
            return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
                                'testing', 'profiles')
        if build:
            return os.path.join(build.topsrcdir, 'testing', 'profiles')
        return os.path.join(here, 'profile_data')

    def start_control_server(self):
        self.control_server = RaptorControlServer(self.results_handler)
        self.control_server.start()

        # for android we must make the control server available to the device
        if self.config['app'] == "geckoview":
            self.log.info(
                "making the raptor control server port available to device")
            _tcp_port = "tcp:%s" % self.control_server.port
            self.device.create_socket_connection('reverse', _tcp_port,
                                                 _tcp_port)

    def get_playback_config(self, test):
        self.config['playback_tool'] = test.get('playback')
        self.log.info("test uses playback tool: %s " %
                      self.config['playback_tool'])
        self.config['playback_binary_manifest'] = test.get(
            'playback_binary_manifest', None)
        _key = 'playback_binary_zip_%s' % self.config['platform']
        self.config['playback_binary_zip'] = test.get(_key, None)
        self.config['playback_pageset_manifest'] = test.get(
            'playback_pageset_manifest', None)
        _key = 'playback_pageset_zip_%s' % self.config['platform']
        self.config['playback_pageset_zip'] = test.get(_key, None)
        self.config['playback_recordings'] = test.get('playback_recordings',
                                                      None)
        self.config['python3_win_manifest'] = test.get('python3_win_manifest',
                                                       None)

    def run_test(self, test, timeout=None):
        self.log.info("starting raptor test: %s" % test['name'])
        self.log.info("test settings: %s" % str(test))
        self.log.info("raptor config: %s" % str(self.config))

        # benchmark-type tests require the benchmark test to be served out
        if test.get('type') == "benchmark":
            self.benchmark = Benchmark(self.config, test)
            benchmark_port = int(self.benchmark.port)
        else:
            benchmark_port = 0

        gen_test_config(self.config['app'], test['name'],
                        self.control_server.port, benchmark_port)

        # for android we must make the benchmarks server available to the device
        if self.config['app'] == "geckoview":
            self.log.info(
                "making the raptor benchmarks server port available to device")
            _tcp_port = "tcp:%s" % benchmark_port
            self.device.create_socket_connection('reverse', _tcp_port,
                                                 _tcp_port)

        # must intall raptor addon each time because we dynamically update some content
        raptor_webext = os.path.join(webext_dir, 'raptor')
        self.log.info("installing webext %s" % raptor_webext)
        self.profile.addons.install(raptor_webext)

        # add test specific preferences
        if test.get("preferences", None) is not None:
            if self.config['app'] == "firefox":
                self.profile.set_preferences(json.loads(test['preferences']))
            else:
                self.log.info("preferences were configured for the test, \
                              but we do not install them on non Firefox browsers."
                              )

        # on firefox we can get an addon id; chrome addon actually is just cmd line arg
        if self.config['app'] in ["firefox", "geckoview"]:
            webext_id = self.profile.addons.addon_details(raptor_webext)['id']

        # some tests require tools to playback the test pages
        if test.get('playback', None) is not None:
            self.get_playback_config(test)
            # startup the playback tool
            self.playback = get_playback(self.config)

        # for geckoview we must copy the profile onto the device and set perms
        if self.config['app'] == "geckoview":
            if not self.device.is_app_installed(self.config['binary']):
                raise Exception('%s is not installed' % self.config['binary'])

            self.log.info("copying firefox profile onto the android device")
            self.device_profile = "/sdcard/raptor-profile"
            if self.device.is_dir(self.device_profile):
                self.device.rm(self.device_profile, recursive=True)

            self.device.mkdir(self.device_profile)
            self.device.push(self.profile.profile, self.device_profile)

            self.log.info("setting permisions to profile dir on the device")
            self.device.chmod(self.device_profile, recursive=True)

            # now start the geckoview app
            self.log.info("starting %s" % self.config['app'])

            extra_args = [
                "-profile", self.device_profile, "--es", "env0",
                "LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6"
            ]

            try:
                # make sure the geckoview app is not running before
                # attempting to start.
                self.device.stop_application(self.config['binary'])
                self.device.launch_activity(self.config['binary'],
                                            "GeckoViewActivity",
                                            extra_args=extra_args,
                                            url='about:blank',
                                            fail_if_running=False)
            except Exception:
                self.log.error("Exception launching %s" %
                               self.config['binary'])
                raise
            self.control_server.device = self.device
            self.control_server.app_name = self.config['binary']

        else:
            # now start the desktop browser
            self.log.info("starting %s" % self.config['app'])

            self.runner.start()
            proc = self.runner.process_handler
            self.output_handler.proc = proc

            self.control_server.browser_proc = proc

        # set our cs flag to indicate we are running the browser/app
        self.control_server._finished = False

        # convert to seconds and account for page cycles
        timeout = int(timeout / 1000) * int(test['page_cycles'])
        try:
            elapsed_time = 0
            while not self.control_server._finished:
                time.sleep(1)
                elapsed_time += 1
                if elapsed_time > (timeout) - 5:  # stop 5 seconds early
                    self.log.info(
                        "application timed out after {} seconds".format(
                            timeout))
                    self.control_server.wait_for_quit()
                    break
        finally:
            if self.config['app'] != "geckoview":
                try:
                    self.runner.check_for_crashes()
                except NotImplementedError:  # not implemented for Chrome
                    pass
            # TODO: if on geckoview is there some cleanup here i.e. check for crashes?

        if self.playback is not None:
            self.playback.stop()

        # remove the raptor webext; as it must be reloaded with each subtest anyway
        # applies to firefox only; chrome the addon is actually just cmd line arg
        if self.config['app'] in ["firefox", "geckoview"]:
            self.log.info("removing webext %s" % raptor_webext)
            self.profile.addons.remove_addon(webext_id)

        if self.config['app'] != "geckoview":
            if self.runner.is_running():
                self.runner.stop()
        # TODO the geckoview app should have been shutdown by this point by the
        # control server, but we can double-check here to make sure

    def process_results(self):
        # when running locally output results in build/raptor.json; when running
        # in production output to a local.json to be turned into tc job artifact
        if self.config.get('run_local', False):
            if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
                raptor_json_path = os.path.join(
                    os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing',
                    'mozharness', 'build', 'raptor.json')
            else:
                raptor_json_path = os.path.join(here, 'raptor.json')
        else:
            raptor_json_path = os.path.join(os.getcwd(), 'local.json')

        self.config['raptor_json_path'] = raptor_json_path
        return self.results_handler.summarize_and_output(self.config)

    def clean_up(self):
        self.control_server.stop()
        if self.config['app'] != "geckoview":
            self.runner.stop()
        elif self.config['app'] == 'geckoview':
            self.log.info('removing reverse socket connections')
            self.device.remove_socket_connections('reverse')
        else:
            pass
        self.log.info("finished")
Example #2
0
class Raptor(object):
    """Container class for Raptor"""

    def __init__(self, app, binary, run_local=False, obj_path=None,
                 gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None,
                 symbols_path=None, host=None, power_test=False, is_release_build=False,
                 debug_mode=False, activity=None):

        # Override the magic --host HOST_IP with the value of the environment variable.
        if host == 'HOST_IP':
            host = os.environ['HOST_IP']
        self.config = {}
        self.config['app'] = app
        self.config['binary'] = binary
        self.config['platform'] = mozinfo.os
        self.config['processor'] = mozinfo.processor
        self.config['run_local'] = run_local
        self.config['obj_path'] = obj_path
        self.config['gecko_profile'] = gecko_profile
        self.config['gecko_profile_interval'] = gecko_profile_interval
        self.config['gecko_profile_entries'] = gecko_profile_entries
        self.config['symbols_path'] = symbols_path
        self.config['host'] = host
        self.config['power_test'] = power_test
        self.config['is_release_build'] = is_release_build
        self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
        self.log = get_default_logger(component='raptor-main')
        self.control_server = None
        self.playback = None
        self.benchmark = None
        self.benchmark_port = 0
        self.gecko_profiler = None
        self.post_startup_delay = 30000
        self.device = None
        self.profile_class = app
        self.firefox_android_apps = ['fennec', 'geckoview', 'refbrow', 'fenix']

        # debug mode is currently only supported when running locally
        self.debug_mode = debug_mode if self.config['run_local'] else False

        # if running debug-mode reduce the pause after browser startup
        if self.debug_mode:
            self.post_startup_delay = 3000
            self.log.info("debug-mode enabled, reducing post-browser startup pause to %d ms"
                          % self.post_startup_delay)

        self.log.info("main raptor init, config is: %s" % str(self.config))

        # create results holder
        self.results_handler = RaptorResultsHandler()

    @property
    def profile_data_dir(self):
        if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
            return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'profiles')
        if build:
            return os.path.join(build.topsrcdir, 'testing', 'profiles')
        return os.path.join(here, 'profile_data')

    def run_test_setup(self, test):
        self.log.info("starting raptor test: %s" % test['name'])
        self.log.info("test settings: %s" % str(test))
        self.log.info("raptor config: %s" % str(self.config))

        if test.get('type') == "benchmark":
            self.serve_benchmark_source(test)

        gen_test_config(self.config['app'],
                        test['name'],
                        self.control_server.port,
                        self.post_startup_delay,
                        host=self.config['host'],
                        b_port=self.benchmark_port,
                        debug_mode=1 if self.debug_mode else 0)

        self.install_raptor_webext()

        if test.get("preferences", None) is not None:
            self.set_browser_test_prefs(test['preferences'])

        # if 'alert_on' was provided in the test INI, add to our config for results/output
        self.config['subtest_alert_on'] = test.get('alert_on', None)

    def set_browser_test_prefs(self, raw_prefs):
        # add test specific preferences
        if self.config['app'] == "firefox":
            self.log.info("setting test-specific browser preferences")
            self.profile.set_preferences(json.loads(raw_prefs))
        else:
            self.log.info("preferences were configured for the test, however \
                          we currently do not install them on non Firefox browsers.")

    def run_test_teardown(self):
        self.check_for_crashes()

        if self.playback is not None:
            self.playback.stop()

        self.remove_raptor_webext()

        # gecko profiling symbolication
        if self.config['gecko_profile'] is True:
            self.gecko_profiler.symbolicate()
            # clean up the temp gecko profiling folders
            self.log.info("cleaning up after gecko profiling")
            self.gecko_profiler.clean()

    def create_browser_profile(self):
        self.profile = create_profile(self.profile_class)

        # Merge in base profiles
        with open(os.path.join(self.profile_data_dir, 'profiles.json'), 'r') as fh:
            base_profiles = json.load(fh)['raptor']

        for name in base_profiles:
            path = os.path.join(self.profile_data_dir, name)
            self.log.info("Merging profile: {}".format(path))
            self.profile.merge(path)

        # add profile dir to our config
        self.config['local_profile_dir'] = self.profile.profile

    def start_control_server(self):
        self.control_server = RaptorControlServer(self.results_handler, self.debug_mode)
        self.control_server.start()

        # for android we must make the control server available to the device
        if self.config['app'] in self.firefox_android_apps and \
                self.config['host'] in ('localhost', '127.0.0.1'):
            self.log.info("making the raptor control server port available to device")
            _tcp_port = "tcp:%s" % self.control_server.port
            self.device.create_socket_connection('reverse', _tcp_port, _tcp_port)

    def get_playback_config(self, test):
        self.config['playback_tool'] = test.get('playback')
        self.log.info("test uses playback tool: %s " % self.config['playback_tool'])
        self.config['playback_binary_manifest'] = test.get('playback_binary_manifest', None)
        _key = 'playback_binary_zip_%s' % self.config['platform']
        self.config['playback_binary_zip'] = test.get(_key, None)
        self.config['playback_pageset_manifest'] = test.get('playback_pageset_manifest', None)
        _key = 'playback_pageset_zip_%s' % self.config['platform']
        self.config['playback_pageset_zip'] = test.get(_key, None)
        self.config['playback_recordings'] = test.get('playback_recordings', None)
        playback_dir = os.path.join(here, 'playback')
        for key in ('playback_pageset_manifest', 'playback_pageset_zip'):
            if self.config.get(key) is None:
                continue
            self.config[key] = os.path.join(playback_dir, self.config[key])
        self.config['custom_script'] = os.path.join(playback_dir, 'alternate-server-replay.py')

    def serve_benchmark_source(self, test):
        # benchmark-type tests require the benchmark test to be served out
        self.benchmark = Benchmark(self.config, test)
        self.benchmark_port = int(self.benchmark.port)

        # for android we must make the benchmarks server available to the device
        if self.config['app'] in self.firefox_android_apps and \
                self.config['host'] in ('localhost', '127.0.0.1'):
            self.log.info("making the raptor benchmarks server port available to device")
            _tcp_port = "tcp:%s" % self.benchmark_port
            self.device.create_socket_connection('reverse', _tcp_port, _tcp_port)

    def install_raptor_webext(self):
        # must intall raptor addon each time because we dynamically update some content
        # the webext is installed into the browser profile
        # note: for chrome the addon is just a list of paths that ultimately are added
        # to the chromium command line '--load-extension' argument
        self.raptor_webext = os.path.join(webext_dir, 'raptor')
        self.log.info("installing webext %s" % self.raptor_webext)
        self.profile.addons.install(self.raptor_webext)

        # on firefox we can get an addon id; chrome addon actually is just cmd line arg
        try:
            self.webext_id = self.profile.addons.addon_details(self.raptor_webext)['id']
        except AttributeError:
            self.webext_id = None

    def remove_raptor_webext(self):
        # remove the raptor webext; as it must be reloaded with each subtest anyway
        self.log.info("removing webext %s" % self.raptor_webext)
        if self.config['app'] in ['firefox', 'geckoview', 'fennec', 'refbrow', 'fenix']:
            self.profile.addons.remove_addon(self.webext_id)

        # for chrome the addon is just a list (appended to cmd line)
        if self.config['app'] in ["chrome", "chrome-android"]:
            self.profile.addons.remove(self.raptor_webext)

    def set_test_browser_prefs(self, test_prefs):
        # if the test has any specific browser prefs specified, set those in browser profiile
        if self.config['app'] == "firefox":
            self.profile.set_preferences(json.loads(test_prefs))
        else:
            self.log.info("preferences were configured for the test, \
                          but we do not install them on non Firefox browsers.")

    def start_playback(self, test):
        # startup the playback tool
        self.get_playback_config(test)
        self.playback = get_playback(self.config, self.device)

        # for android we must make the playback server available to the device
        if self.config['app'] in self.firefox_android_apps and self.config['host'] \
                in ('localhost', '127.0.0.1'):
            self.log.info("making the raptor playback server port available to device")
            _tcp_port = "tcp:8080"
            self.device.create_socket_connection('reverse', _tcp_port, _tcp_port)

    def delete_proxy_settings_from_profile(self):
        # Must delete the proxy settings from the profile if running
        # the test with a host different from localhost.
        userjspath = os.path.join(self.profile.profile, 'user.js')
        with open(userjspath) as userjsfile:
            prefs = userjsfile.readlines()
        prefs = [pref for pref in prefs if 'network.proxy' not in pref]
        with open(userjspath, 'w') as userjsfile:
            userjsfile.writelines(prefs)

    def _init_gecko_profiling(self, test):
        self.log.info("initializing gecko profiler")
        upload_dir = os.getenv('MOZ_UPLOAD_DIR')
        if not upload_dir:
            self.log.critical("Profiling ignored because MOZ_UPLOAD_DIR was not set")
        else:
            self.gecko_profiler = GeckoProfile(upload_dir,
                                               self.config,
                                               test)

    def wait_for_test_finish(self, test, timeout):
        # convert timeout to seconds and account for page cycles
        timeout = int(timeout / 1000) * int(test['page_cycles'])
        # account for the pause the raptor webext runner takes after browser startup
        timeout += (int(self.post_startup_delay / 1000) + 3)

        # if geckoProfile enabled, give browser more time for profiling
        if self.config['gecko_profile'] is True:
            timeout += 5 * 60

        elapsed_time = 0
        while not self.control_server._finished:
            time.sleep(1)
            # we only want to force browser-shutdown on timeout if not in debug mode;
            # in debug-mode we leave the browser running (require manual shutdown)
            if not self.debug_mode:
                elapsed_time += 1
                if elapsed_time > (timeout) - 5:  # stop 5 seconds early
                    self.log.info("application timed out after {} seconds".format(timeout))
                    self.control_server.wait_for_quit()
                    break

    def process_results(self, test_names):
        # when running locally output results in build/raptor.json; when running
        # in production output to a local.json to be turned into tc job artifact
        if self.config.get('run_local', False):
            if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
                raptor_json_path = os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
                                                'testing', 'mozharness', 'build', 'raptor.json')
            else:
                raptor_json_path = os.path.join(here, 'raptor.json')
        else:
            raptor_json_path = os.path.join(os.getcwd(), 'local.json')

        self.config['raptor_json_path'] = raptor_json_path
        return self.results_handler.summarize_and_output(self.config, test_names)

    def get_page_timeout_list(self):
        return self.results_handler.page_timeout_list

    def clean_up(self):
        self.control_server.stop()
        if self.config['app'] not in self.firefox_android_apps:
            self.runner.stop()
        elif self.config['app'] in self.firefox_android_apps:
            self.log.info('removing reverse socket connections')
            self.device.remove_socket_connections('reverse')
        else:
            pass
        self.log.info("finished")
Example #3
0
class Raptor(object):
    """Container class for Raptor"""
    def __init__(self, app, binary, run_local=False, obj_path=None):
        self.config = {}
        self.config['app'] = app
        self.config['binary'] = binary
        self.config['platform'] = mozinfo.os
        self.config['processor'] = mozinfo.processor
        self.config['run_local'] = run_local
        self.config['obj_path'] = obj_path
        self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
        self.log = get_default_logger(component='raptor-main')
        self.control_server = None
        self.playback = None
        self.benchmark = None

        # Create the profile
        self.profile = create_profile(self.config['app'])

        # Merge in base profiles
        with open(os.path.join(self.profile_data_dir, 'profiles.json'),
                  'r') as fh:
            base_profiles = json.load(fh)['raptor']

        for name in base_profiles:
            path = os.path.join(self.profile_data_dir, name)
            self.log.info("Merging profile: {}".format(path))
            self.profile.merge(path)

        # create results holder
        self.results_handler = RaptorResultsHandler()

        # Create the runner
        self.output_handler = OutputHandler()
        process_args = {
            'processOutputLine': [self.output_handler],
        }
        runner_cls = runners[app]
        self.runner = runner_cls(binary,
                                 profile=self.profile,
                                 process_args=process_args)

        self.log.info("raptor config: %s" % str(self.config))

    @property
    def profile_data_dir(self):
        if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
            return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
                                'testing', 'profiles')
        if build:
            return os.path.join(build.topsrcdir, 'testing', 'profiles')
        return os.path.join(here, 'profile_data')

    def start_control_server(self):
        self.control_server = RaptorControlServer(self.results_handler)
        self.control_server.start()

    def get_playback_config(self, test):
        self.config['playback_tool'] = test.get('playback')
        self.log.info("test uses playback tool: %s " %
                      self.config['playback_tool'])
        self.config['playback_binary_manifest'] = test.get(
            'playback_binary_manifest', None)
        _key = 'playback_binary_zip_%s' % self.config['platform']
        self.config['playback_binary_zip'] = test.get(_key, None)
        self.config['playback_pageset_manifest'] = test.get(
            'playback_pageset_manifest', None)
        _key = 'playback_pageset_zip_%s' % self.config['platform']
        self.config['playback_pageset_zip'] = test.get(_key, None)
        self.config['playback_recordings'] = test.get('playback_recordings',
                                                      None)
        self.config['python3_win_manifest'] = test.get('python3_win_manifest',
                                                       None)

    def run_test(self, test, timeout=None):
        self.log.info("starting raptor test: %s" % test['name'])
        self.log.info("test settings: %s" % str(test))
        self.log.info("raptor config: %s" % str(self.config))

        # benchmark-type tests require the benchmark test to be served out
        if test.get('type') == "benchmark":
            self.benchmark = Benchmark(self.config, test)
            benchmark_port = int(self.benchmark.port)
        else:
            benchmark_port = 0

        gen_test_config(self.config['app'], test['name'],
                        self.control_server.port, benchmark_port)

        # must intall raptor addon each time because we dynamically update some content
        raptor_webext = os.path.join(webext_dir, 'raptor')
        self.log.info("installing webext %s" % raptor_webext)
        self.profile.addons.install(raptor_webext)
        # on firefox we can get an addon id; chrome addon actually is just cmd line arg
        if self.config['app'] == "firefox":
            webext_id = self.profile.addons.addon_details(raptor_webext)['id']

        # some tests require tools to playback the test pages
        if test.get('playback', None) is not None:
            self.get_playback_config(test)
            # startup the playback tool
            self.playback = get_playback(self.config)

        self.runner.start()

        proc = self.runner.process_handler
        self.output_handler.proc = proc
        self.control_server.browser_proc = proc
        self.control_server._finished = False

        # convert to seconds and account for page cycles
        timeout = int(timeout / 1000) * int(test['page_cycles'])
        try:
            elapsed_time = 0
            while not self.control_server._finished:
                time.sleep(1)
                elapsed_time += 1
                if elapsed_time > (timeout) - 5:  # stop 5 seconds early
                    self.control_server.wait_for_quit()
                    break
        finally:
            try:
                self.runner.check_for_crashes()
            except NotImplementedError:  # not implemented for Chrome
                pass

        if self.playback is not None:
            self.playback.stop()

        # remove the raptor webext; as it must be reloaded with each subtest anyway
        # applies to firefox only; chrome the addon is actually just cmd line arg
        if self.config['app'] == "firefox":
            self.log.info("removing webext %s" % raptor_webext)
            self.profile.addons.remove_addon(webext_id)

        if self.runner.is_running():
            self.log.info(
                "Application timed out after {} seconds".format(timeout))
            self.runner.stop()

    def process_results(self):
        # when running locally output results in build/raptor.json; when running
        # in production output to a local.json to be turned into tc job artifact
        if self.config.get('run_local', False):
            if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
                raptor_json_path = os.path.join(
                    os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing',
                    'mozharness', 'build', 'raptor.json')
            else:
                raptor_json_path = os.path.join(here, 'raptor.json')
        else:
            raptor_json_path = os.path.join(os.getcwd(), 'local.json')

        self.config['raptor_json_path'] = raptor_json_path
        return self.results_handler.summarize_and_output(self.config)

    def clean_up(self):
        self.control_server.stop()
        self.runner.stop()
        self.log.info("finished")
Example #4
0
class Raptor(object):
    """Container class for Raptor"""
    def __init__(self,
                 app,
                 binary,
                 run_local=False,
                 obj_path=None,
                 gecko_profile=False,
                 gecko_profile_interval=None,
                 gecko_profile_entries=None,
                 symbols_path=None,
                 host=None,
                 is_release_build=False,
                 debug_mode=False):
        self.config = {}
        self.config['app'] = app
        self.config['binary'] = binary
        self.config['platform'] = mozinfo.os
        self.config['processor'] = mozinfo.processor
        self.config['run_local'] = run_local
        self.config['obj_path'] = obj_path
        self.config['gecko_profile'] = gecko_profile
        self.config['gecko_profile_interval'] = gecko_profile_interval
        self.config['gecko_profile_entries'] = gecko_profile_entries
        self.config['symbols_path'] = symbols_path
        self.config['host'] = host
        self.config['is_release_build'] = is_release_build
        self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
        self.log = get_default_logger(component='raptor-main')
        self.control_server = None
        self.playback = None
        self.benchmark = None
        self.gecko_profiler = None
        self.post_startup_delay = 30000
        self.device = None

        # debug mode is currently only supported when running locally
        self.debug_mode = debug_mode if self.config['run_local'] else False

        # if running debug-mode reduce the pause after browser startup
        if self.debug_mode:
            self.post_startup_delay = 3000
            self.log.info(
                "debug-mode enabled, reducing post-browser startup pause to %d ms"
                % self.post_startup_delay)

        # Create the profile; for geckoview we want a firefox profile type
        if self.config['app'] == 'geckoview':
            self.profile = create_profile('firefox')
        else:
            self.profile = create_profile(self.config['app'])

        # Merge in base profiles
        with open(os.path.join(self.profile_data_dir, 'profiles.json'),
                  'r') as fh:
            base_profiles = json.load(fh)['raptor']

        for name in base_profiles:
            path = os.path.join(self.profile_data_dir, name)
            self.log.info("Merging profile: {}".format(path))
            self.profile.merge(path)

        # add profile dir to our config
        self.config['local_profile_dir'] = self.profile.profile

        # create results holder
        self.results_handler = RaptorResultsHandler()

        # when testing desktop browsers we use mozrunner to start the browser; when
        # testing on android (i.e. geckoview) we use mozdevice to control the device app

        if self.config['app'] == "geckoview":
            # create the android device handler; it gets initiated and sets up adb etc
            self.log.info("creating android device handler using mozdevice")
            self.device = ADBAndroid(verbose=True)
            self.device.clear_logcat()
        else:
            # create the desktop browser runner
            self.log.info("creating browser runner using mozrunner")
            self.output_handler = OutputHandler()
            process_args = {
                'processOutputLine': [self.output_handler],
            }
            runner_cls = runners[app]
            self.runner = runner_cls(binary,
                                     profile=self.profile,
                                     process_args=process_args)

        self.log.info("raptor config: %s" % str(self.config))

    @property
    def profile_data_dir(self):
        if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
            return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
                                'testing', 'profiles')
        if build:
            return os.path.join(build.topsrcdir, 'testing', 'profiles')
        return os.path.join(here, 'profile_data')

    def start_control_server(self):
        self.control_server = RaptorControlServer(self.results_handler,
                                                  self.debug_mode)
        self.control_server.start()

        # for android we must make the control server available to the device
        if self.config['app'] == "geckoview" and self.config['host'] in (
                'localhost', '127.0.0.1'):
            self.log.info(
                "making the raptor control server port available to device")
            _tcp_port = "tcp:%s" % self.control_server.port
            self.device.create_socket_connection('reverse', _tcp_port,
                                                 _tcp_port)

    def get_playback_config(self, test):
        self.config['playback_tool'] = test.get('playback')
        self.log.info("test uses playback tool: %s " %
                      self.config['playback_tool'])
        self.config['playback_binary_manifest'] = test.get(
            'playback_binary_manifest', None)
        _key = 'playback_binary_zip_%s' % self.config['platform']
        self.config['playback_binary_zip'] = test.get(_key, None)
        self.config['playback_pageset_manifest'] = test.get(
            'playback_pageset_manifest', None)
        _key = 'playback_pageset_zip_%s' % self.config['platform']
        self.config['playback_pageset_zip'] = test.get(_key, None)
        self.config['playback_recordings'] = test.get('playback_recordings',
                                                      None)
        self.config['python3_win_manifest'] = test.get('python3_win_manifest',
                                                       None)

    def run_test(self, test, timeout=None):
        self.log.info("starting raptor test: %s" % test['name'])
        self.log.info("test settings: %s" % str(test))
        self.log.info("raptor config: %s" % str(self.config))

        # benchmark-type tests require the benchmark test to be served out
        if test.get('type') == "benchmark":
            self.benchmark = Benchmark(self.config, test)
            benchmark_port = int(self.benchmark.port)

            # for android we must make the benchmarks server available to the device
            if self.config['app'] == "geckoview" and self.config['host'] \
                    in ('localhost', '127.0.0.1'):
                self.log.info(
                    "making the raptor benchmarks server port available to device"
                )
                _tcp_port = "tcp:%s" % benchmark_port
                self.device.create_socket_connection('reverse', _tcp_port,
                                                     _tcp_port)
        else:
            benchmark_port = 0

        gen_test_config(self.config['app'],
                        test['name'],
                        self.control_server.port,
                        self.post_startup_delay,
                        host=self.config['host'],
                        b_port=benchmark_port,
                        debug_mode=1 if self.debug_mode else 0)

        # must intall raptor addon each time because we dynamically update some content
        # note: for chrome the addon is just a list of paths that ultimately are added
        # to the chromium command line '--load-extension' argument
        raptor_webext = os.path.join(webext_dir, 'raptor')
        self.log.info("installing webext %s" % raptor_webext)
        self.profile.addons.install(raptor_webext)

        # add test specific preferences
        if test.get("preferences", None) is not None:
            if self.config['app'] == "firefox":
                self.profile.set_preferences(json.loads(test['preferences']))
            else:
                self.log.info("preferences were configured for the test, \
                              but we do not install them on non Firefox browsers."
                              )

        # on firefox we can get an addon id; chrome addon actually is just cmd line arg
        if self.config['app'] in ["firefox", "geckoview"]:
            webext_id = self.profile.addons.addon_details(raptor_webext)['id']

        # for android/geckoview, create a top-level raptor folder on the device
        # sdcard; if it already exists remove it so we start fresh each time
        if self.config['app'] == "geckoview":
            self.device_raptor_dir = "/sdcard/raptor"
            self.config['device_raptor_dir'] = self.device_raptor_dir
            if self.device.is_dir(self.device_raptor_dir):
                self.log.info("deleting existing device raptor dir: %s" %
                              self.device_raptor_dir)
                self.device.rm(self.device_raptor_dir, recursive=True)
            self.log.info("creating raptor folder on sdcard: %s" %
                          self.device_raptor_dir)
            self.device.mkdir(self.device_raptor_dir)
            self.device.chmod(self.device_raptor_dir, recursive=True)

        # some tests require tools to playback the test pages
        if test.get('playback', None) is not None:
            self.get_playback_config(test)
            # startup the playback tool
            self.playback = get_playback(self.config, self.device)

            # for android we must make the playback server available to the device
            if self.config['app'] == "geckoview" and self.config['host'] \
                    in ('localhost', '127.0.0.1'):
                self.log.info(
                    "making the raptor playback server port available to device"
                )
                _tcp_port = "tcp:8080"
                self.device.create_socket_connection('reverse', _tcp_port,
                                                     _tcp_port)

        if self.config['app'] in ("geckoview", "firefox") and \
           self.config['host'] not in ('localhost', '127.0.0.1'):
            # Must delete the proxy settings from the profile if running
            # the test with a host different from localhost.
            userjspath = os.path.join(self.profile.profile, 'user.js')
            with open(userjspath) as userjsfile:
                prefs = userjsfile.readlines()
            prefs = [pref for pref in prefs if 'network.proxy' not in pref]
            with open(userjspath, 'w') as userjsfile:
                userjsfile.writelines(prefs)

        # for geckoview/android pageload playback we can't use a policy to turn on the
        # proxy; we need to set prefs instead; note that the 'host' may be different
        # than '127.0.0.1' so we must set the prefs accordingly
        if self.config['app'] == "geckoview" and test.get('playback',
                                                          None) is not None:
            self.log.info(
                "setting profile prefs to turn on the geckoview browser proxy")
            no_proxies_on = "localhost, 127.0.0.1, %s" % self.config['host']
            proxy_prefs = {}
            proxy_prefs["network.proxy.type"] = 1
            proxy_prefs["network.proxy.http"] = self.config['host']
            proxy_prefs["network.proxy.http_port"] = 8080
            proxy_prefs["network.proxy.ssl"] = self.config['host']
            proxy_prefs["network.proxy.ssl_port"] = 8080
            proxy_prefs["network.proxy.no_proxies_on"] = no_proxies_on
            self.profile.set_preferences(proxy_prefs)

        # now some final settings, and then startup of the browser under test
        if self.config['app'] == "geckoview":
            # for android/geckoview we must copy the profile onto the device and set perms
            if not self.device.is_app_installed(self.config['binary']):
                raise Exception('%s is not installed' % self.config['binary'])
            self.device_profile = os.path.join(self.device_raptor_dir,
                                               "profile")
            if self.device.is_dir(self.device_profile):
                self.log.info("deleting existing device profile folder: %s" %
                              self.device_profile)
                self.device.rm(self.device_profile, recursive=True)
            self.log.info("creating profile folder on device: %s" %
                          self.device_profile)
            self.device.mkdir(self.device_profile)
            self.log.info("copying firefox profile onto the device")
            self.log.info("note: the profile folder being copied is: %s" %
                          self.profile.profile)
            self.log.info(
                'the adb push cmd copies that profile dir to a new temp dir before copy'
            )
            self.device.push(self.profile.profile, self.device_profile)
            self.device.chmod(self.device_profile, recursive=True)

            # now start the geckoview app
            self.log.info("starting %s" % self.config['app'])

            extra_args = [
                "-profile", self.device_profile, "--es", "env0",
                "LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6"
            ]

            try:
                # make sure the geckoview app is not running before
                # attempting to start.
                self.device.stop_application(self.config['binary'])
                self.device.launch_activity(self.config['binary'],
                                            "GeckoViewActivity",
                                            extra_args=extra_args,
                                            url='about:blank',
                                            e10s=True,
                                            fail_if_running=False)
            except Exception:
                self.log.error("Exception launching %s" %
                               self.config['binary'])
                raise
            self.control_server.device = self.device
            self.control_server.app_name = self.config['binary']

        else:
            # For Firefox we need to set
            # MOZ_DISABLE_NONLOCAL_CONNECTIONS=1 env var before
            # startup when testing release builds from mozilla-beta or
            # mozilla-release. This is because of restrictions on
            # release builds that require webextensions to be signed
            # unless MOZ_DISABLE_NONLOCAL_CONNECTIONS is set to '1'.
            if self.config['app'] == "firefox" and self.config[
                    'is_release_build']:
                self.log.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=1")
                os.environ['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = "1"

            # if running debug-mode, tell Firefox to open the browser console on startup
            # for google chrome, open the devtools on the raptor test tab
            if self.debug_mode:
                if self.config['app'] == "firefox":
                    self.runner.cmdargs.extend(['-jsconsole'])
                if self.config['app'] == "chrome":
                    self.runner.cmdargs.extend(
                        ['--auto-open-devtools-for-tabs'])

            # now start the desktop browser
            self.log.info("starting %s" % self.config['app'])

            # if running a pageload test on google chrome, add the cmd line options
            # to turn on the proxy and ignore security certificate errors
            # if using host localhost, 127.0.0.1.
            if self.config['app'] == "chrome" and test.get('playback',
                                                           None) is not None:
                chrome_args = [
                    '--proxy-server="http=127.0.0.1:8080;' +
                    'https=127.0.0.1:8080;ssl=127.0.0.1:8080"',
                    '--ignore-certificate-errors'
                ]
                if self.config['host'] not in ('localhost', '127.0.0.1'):
                    chrome_args[0] = chrome_args[0].replace(
                        '127.0.0.1', self.config['host'])
                if ' '.join(chrome_args) not in ' '.join(self.runner.cmdargs):
                    self.runner.cmdargs.extend(chrome_args)

            self.runner.start()
            proc = self.runner.process_handler
            self.output_handler.proc = proc

            self.control_server.browser_proc = proc

            # pageload tests need to be able to access non-local connections via mitmproxy
            if self.config['app'] == "firefox" and self.config['is_release_build'] and \
               test.get('playback', None) is not None:
                self.log.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=0")
                os.environ['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = "0"

            # if geckoProfile is enabled, initialize it
            if self.config['gecko_profile'] is True:
                self._init_gecko_profiling(test)
                # tell the control server the gecko_profile dir; the control server will
                # receive the actual gecko profiles from the web ext and will write them
                # to disk; then profiles are picked up by gecko_profile.symbolicate
                self.control_server.gecko_profile_dir = self.gecko_profiler.gecko_profile_dir

        # set our cs flag to indicate we are running the browser/app
        self.control_server._finished = False

        # convert to seconds and account for page cycles
        timeout = int(timeout / 1000) * int(test['page_cycles'])
        # account for the pause the raptor webext runner takes after browser startup
        timeout += int(self.post_startup_delay / 1000)

        # if geckoProfile enabled, give browser more time for profiling
        if self.config['gecko_profile'] is True:
            timeout += 5 * 60

        try:
            elapsed_time = 0
            while not self.control_server._finished:
                time.sleep(1)
                # we only want to force browser-shutdown on timeout if not in debug mode;
                # in debug-mode we leave the browser running (require manual shutdown)
                if not self.debug_mode:
                    elapsed_time += 1
                    if elapsed_time > (timeout) - 5:  # stop 5 seconds early
                        self.log.info(
                            "application timed out after {} seconds".format(
                                timeout))
                        self.control_server.wait_for_quit()
                        break
        finally:
            if self.config['app'] != "geckoview":
                try:
                    self.runner.check_for_crashes()
                except NotImplementedError:  # not implemented for Chrome
                    pass
            # TODO: if on geckoview is there some cleanup here i.e. check for crashes?

        if self.playback is not None:
            self.playback.stop()

        # remove the raptor webext; as it must be reloaded with each subtest anyway
        self.log.info("removing webext %s" % raptor_webext)
        if self.config['app'] in ["firefox", "geckoview"]:
            self.profile.addons.remove_addon(webext_id)

        # for chrome the addon is just a list (appended to cmd line)
        if self.config['app'] in ["chrome", "chrome-android"]:
            self.profile.addons.remove(raptor_webext)

        # gecko profiling symbolication
        if self.config['gecko_profile'] is True:
            self.gecko_profiler.symbolicate()
            # clean up the temp gecko profiling folders
            self.log.info("cleaning up after gecko profiling")
            self.gecko_profiler.clean()

        # browser should be closed by now but this is a backup-shutdown (if not in debug-mode)
        if not self.debug_mode:
            if self.config['app'] != "geckoview":
                if self.runner.is_running():
                    self.runner.stop()
            # TODO the geckoview app should have been shutdown by this point by the
            # control server, but we can double-check here to make sure
        else:
            # in debug mode, and running locally, leave the browser running
            if self.config['run_local']:
                self.log.info(
                    "* debug-mode enabled - please shutdown the browser manually..."
                )
                self.runner.wait(timeout=None)

    def _init_gecko_profiling(self, test):
        self.log.info("initializing gecko profiler")
        upload_dir = os.getenv('MOZ_UPLOAD_DIR')
        if not upload_dir:
            self.log.critical(
                "Profiling ignored because MOZ_UPLOAD_DIR was not set")
        else:
            self.gecko_profiler = GeckoProfile(upload_dir, self.config, test)

    def process_results(self):
        # when running locally output results in build/raptor.json; when running
        # in production output to a local.json to be turned into tc job artifact
        if self.config.get('run_local', False):
            if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
                raptor_json_path = os.path.join(
                    os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing',
                    'mozharness', 'build', 'raptor.json')
            else:
                raptor_json_path = os.path.join(here, 'raptor.json')
        else:
            raptor_json_path = os.path.join(os.getcwd(), 'local.json')

        self.config['raptor_json_path'] = raptor_json_path
        return self.results_handler.summarize_and_output(self.config)

    def get_page_timeout_list(self):
        return self.results_handler.page_timeout_list

    def clean_up(self):
        self.control_server.stop()
        if self.config['app'] != "geckoview":
            self.runner.stop()
        elif self.config['app'] == 'geckoview':
            self.log.info('removing reverse socket connections')
            self.device.remove_socket_connections('reverse')
        else:
            pass
        self.log.info("finished")
Example #5
0
class Raptor(object):
    """Container class for Raptor"""
    def __init__(self, app, binary):
        self.config = {}
        self.config['app'] = app
        self.config['binary'] = binary
        self.config['platform'] = mozinfo.os

        self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
        self.log = get_default_logger(component='raptor')
        self.addons_installed = False
        self.control_server = None
        self.playback = None

        # Create the profile
        self.profile = create_profile(self.config['app'])

        # Merge in base profiles
        with open(os.path.join(self.profile_data_dir, 'profiles.json'),
                  'r') as fh:
            base_profiles = json.load(fh)['raptor']

        for name in base_profiles:
            path = os.path.join(self.profile_data_dir, name)
            self.log.info("Merging profile: {}".format(path))
            self.profile.merge(path)

        # create results holder
        self.results_handler = RaptorResultsHandler()

        # Create the runner
        self.output_handler = OutputHandler()
        process_args = {
            'processOutputLine': [self.output_handler],
        }
        runner_cls = runners[app]
        self.runner = runner_cls(binary,
                                 profile=self.profile,
                                 process_args=process_args)

    @property
    def profile_data_dir(self):
        if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
            return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
                                'testing', 'profiles')
        if build:
            return os.path.join(build.topsrcdir, 'testing', 'profiles')
        return os.path.join(here, 'profile_data')

    def start_control_server(self):
        self.control_server = RaptorControlServer(self.results_handler)
        self.control_server.start()

    def get_playback_config(self, test):
        self.config['playback_tool'] = test.get('playback')
        self.log.info("test uses playback tool: %s " %
                      self.config['playback_tool'])
        self.config['playback_binary_manifest'] = test.get(
            'playback_binary_manifest', None)
        _key = 'playback_binary_zip_%s' % self.config['platform']
        self.config['playback_binary_zip'] = test.get(_key, None)
        self.config['playback_pageset_manifest'] = test.get(
            'playback_pageset_manifest', None)
        _key = 'playback_pageset_zip_%s' % self.config['platform']
        self.config['playback_pageset_zip'] = test.get(_key, None)
        self.config['playback_recordings'] = test.get('playback_recordings',
                                                      None)

    def run_test(self, test, timeout=None):
        self.log.info("starting raptor test: %s" % test['name'])
        gen_test_config(self.config['app'], test['name'],
                        self.control_server.port)

        # must intall raptor addon each time because we dynamically update some content
        raptor_webext = os.path.join(webext_dir, 'raptor')
        self.log.info("installing webext %s" % raptor_webext)
        self.profile.addons.install(raptor_webext)
        webext_id = self.profile.addons.addon_details(raptor_webext)['id']

        # some tests require tools to playback the test pages
        if test.get('playback', None) is not None:
            self.get_playback_config(test)
            # startup the playback tool
            self.playback = get_playback(self.config)

        self.runner.start()

        proc = self.runner.process_handler
        self.output_handler.proc = proc
        self.control_server.browser_proc = proc

        try:
            self.runner.wait(timeout)
        finally:
            try:
                self.runner.check_for_crashes()
            except NotImplementedError:  # not implemented for Chrome
                pass

        if self.playback is not None:
            self.playback.stop()

        # remove the raptor webext; as it must be reloaded with each subtest anyway
        self.log.info("removing webext %s" % raptor_webext)
        self.profile.addons.remove_addon(webext_id)

        if self.runner.is_running():
            self.log("Application timed out after {} seconds".format(timeout))
            self.runner.stop()

    def process_results(self):
        return self.results_handler.summarize_and_output(self.config)

    def clean_up(self):
        self.control_server.stop()
        self.runner.stop()
        self.log.info("finished")
Example #6
0
class Raptor(object):
    """Container class for Raptor"""
    def __init__(self,
                 app,
                 binary,
                 run_local=False,
                 obj_path=None,
                 gecko_profile=False,
                 gecko_profile_interval=None,
                 gecko_profile_entries=None,
                 symbols_path=None,
                 host=None,
                 power_test=False,
                 memory_test=False,
                 is_release_build=False,
                 debug_mode=False,
                 post_startup_delay=None,
                 interrupt_handler=None,
                 **kwargs):

        # Override the magic --host HOST_IP with the value of the environment variable.
        if host == 'HOST_IP':
            host = os.environ['HOST_IP']

        self.config = {
            'app': app,
            'binary': binary,
            'platform': mozinfo.os,
            'processor': mozinfo.processor,
            'run_local': run_local,
            'obj_path': obj_path,
            'gecko_profile': gecko_profile,
            'gecko_profile_interval': gecko_profile_interval,
            'gecko_profile_entries': gecko_profile_entries,
            'symbols_path': symbols_path,
            'host': host,
            'power_test': power_test,
            'memory_test': memory_test,
            'is_release_build': is_release_build,
            'enable_control_server_wait': memory_test,
        }

        self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
        self.log = get_default_logger(component='raptor-main')
        self.control_server = None
        self.playback = None
        self.benchmark = None
        self.benchmark_port = 0
        self.gecko_profiler = None
        self.post_startup_delay = post_startup_delay
        self.device = None
        self.profile_class = app
        self.firefox_android_apps = FIREFOX_ANDROID_APPS
        self.interrupt_handler = interrupt_handler

        # debug mode is currently only supported when running locally
        self.debug_mode = debug_mode if self.config['run_local'] else False

        # if running debug-mode reduce the pause after browser startup
        if self.debug_mode:
            self.post_startup_delay = min(self.post_startup_delay, 3000)
            self.log.info(
                "debug-mode enabled, reducing post-browser startup pause to %d ms"
                % self.post_startup_delay)

        self.log.info("main raptor init, config is: %s" % str(self.config))

        # create results holder
        self.results_handler = RaptorResultsHandler()

    @property
    def profile_data_dir(self):
        if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
            return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
                                'testing', 'profiles')
        if build:
            return os.path.join(build.topsrcdir, 'testing', 'profiles')
        return os.path.join(here, 'profile_data')

    def check_for_crashes(self):
        raise NotImplementedError

    def run_test_setup(self, test):
        self.log.info("starting raptor test: %s" % test['name'])
        self.log.info("test settings: %s" % str(test))
        self.log.info("raptor config: %s" % str(self.config))

        if test.get('type') == "benchmark":
            self.serve_benchmark_source(test)

        gen_test_config(
            self.config['app'],
            test['name'],
            self.control_server.port,
            self.post_startup_delay,
            host=self.config['host'],
            b_port=self.benchmark_port,
            debug_mode=1 if self.debug_mode else 0,
            browser_cycle=test.get('browser_cycle', 1),
        )

        self.install_raptor_webext()

        if test.get("preferences") is not None:
            self.set_browser_test_prefs(test['preferences'])

        # if 'alert_on' was provided in the test INI, add to our config for results/output
        self.config['subtest_alert_on'] = test.get('alert_on')

    def run_test_teardown(self):
        self.check_for_crashes()

        if self.playback is not None:
            self.playback.stop()

        self.remove_raptor_webext()

        # gecko profiling symbolication
        if self.config['gecko_profile'] is True:
            self.gecko_profiler.symbolicate()
            # clean up the temp gecko profiling folders
            self.log.info("cleaning up after gecko profiling")
            self.gecko_profiler.clean()

    def set_browser_test_prefs(self, raw_prefs):
        # add test specific preferences
        self.log.info("preferences were configured for the test, however \
                        we currently do not install them on non Firefox browsers."
                      )

    def create_browser_profile(self):
        self.profile = create_profile(self.profile_class)

        # Merge in base profiles
        with open(os.path.join(self.profile_data_dir, 'profiles.json'),
                  'r') as fh:
            base_profiles = json.load(fh)['raptor']

        for name in base_profiles:
            path = os.path.join(self.profile_data_dir, name)
            self.log.info("Merging profile: {}".format(path))
            self.profile.merge(path)

        # add profile dir to our config
        self.config['local_profile_dir'] = self.profile.profile

    def start_control_server(self):
        self.control_server = RaptorControlServer(self.results_handler,
                                                  self.debug_mode)
        self.control_server.start()

        if self.config['enable_control_server_wait']:
            self.control_server_wait_set(
                'webext_status/__raptor_shutdownBrowser')

    def get_playback_config(self, test):
        platform = self.config['platform']
        playback_dir = os.path.join(here, 'playback')

        self.config.update({
            'playback_tool':
            test.get('playback'),
            'playback_version':
            test.get('playback_version', "2.0.2"),
            'playback_binary_zip':
            test.get('playback_binary_zip_%s' % platform),
            'playback_pageset_zip':
            test.get('playback_pageset_zip_%s' % platform),
            'playback_binary_manifest':
            test.get('playback_binary_manifest'),
            'playback_pageset_manifest':
            test.get('playback_pageset_manifest'),
        })
        for key in ('playback_pageset_manifest', 'playback_pageset_zip'):
            if self.config.get(key) is None:
                continue
            self.config[key] = os.path.join(playback_dir, self.config[key])

        self.log.info("test uses playback tool: %s " %
                      self.config['playback_tool'])

    def serve_benchmark_source(self, test):
        # benchmark-type tests require the benchmark test to be served out
        self.benchmark = Benchmark(self.config, test)
        self.benchmark_port = int(self.benchmark.port)

    def install_raptor_webext(self):
        # must intall raptor addon each time because we dynamically update some content
        # the webext is installed into the browser profile
        # note: for chrome the addon is just a list of paths that ultimately are added
        # to the chromium command line '--load-extension' argument
        self.raptor_webext = os.path.join(webext_dir, 'raptor')
        self.log.info("installing webext %s" % self.raptor_webext)
        self.profile.addons.install(self.raptor_webext)

        # on firefox we can get an addon id; chrome addon actually is just cmd line arg
        try:
            self.webext_id = self.profile.addons.addon_details(
                self.raptor_webext)['id']
        except AttributeError:
            self.webext_id = None

    def remove_raptor_webext(self):
        # remove the raptor webext; as it must be reloaded with each subtest anyway
        self.log.info("removing webext %s" % self.raptor_webext)
        if self.config['app'] in [
                'firefox', 'geckoview', 'fennec', 'refbrow', 'fenix'
        ]:
            self.profile.addons.remove_addon(self.webext_id)

        # for chrome the addon is just a list (appended to cmd line)
        chrome_apps = CHROMIUM_DISTROS + ["chrome-android", "chromium-android"]
        if self.config['app'] in chrome_apps:
            self.profile.addons.remove(self.raptor_webext)

    def get_proxy_command_for_mitm(self, test, version):
        # Generate Mitmproxy playback args
        script = os.path.join(here, "playback",
                              "alternate-server-replay-{}.py".format(version))
        recordings = test.get("playback_recordings")
        if recordings:
            recording_paths = []
            proxy_dir = self.playback.mozproxy_dir
            for recording in recordings.split():
                if not recording:
                    continue
                recording_paths.append(os.path.join(proxy_dir, recording))

        # this part is platform-specific
        if mozinfo.os == "win":
            script = script.replace("\\", "\\\\\\")
            recording_paths = [
                recording_path.replace("\\", "\\\\\\")
                for recording_path in recording_paths
            ]

        if version == "2.0.2":
            self.playback.config['playback_tool_args'] = [
                "--replay-kill-extra", "--script",
                '""{} {}""'.format(script, " ".join(recording_paths))
            ]
        elif version == "4.0.4":
            self.playback.config['playback_tool_args'] = [
                "--scripts", script, "--set",
                "server_replay={}".format(" ".join(recording_paths))
            ]
        else:
            raise Exception("Mitmproxy version is unknown!")

    def start_playback(self, test):
        # creating the playback tool
        self.get_playback_config(test)
        self.playback = get_playback(self.config, self.device)

        self.get_proxy_command_for_mitm(test, self.config['playback_version'])

        # let's start it!
        self.playback.start()

    def delete_proxy_settings_from_profile(self):
        # Must delete the proxy settings from the profile if running
        # the test with a host different from localhost.
        userjspath = os.path.join(self.profile.profile, 'user.js')
        with open(userjspath) as userjsfile:
            prefs = userjsfile.readlines()
        prefs = [pref for pref in prefs if 'network.proxy' not in pref]
        with open(userjspath, 'w') as userjsfile:
            userjsfile.writelines(prefs)

    def _init_gecko_profiling(self, test):
        self.log.info("initializing gecko profiler")
        upload_dir = os.getenv('MOZ_UPLOAD_DIR')
        if not upload_dir:
            self.log.critical(
                "Profiling ignored because MOZ_UPLOAD_DIR was not set")
        else:
            self.gecko_profiler = GeckoProfile(upload_dir, self.config, test)

    def wait_for_test_finish(self, test, timeout):
        # convert timeout to seconds and account for page cycles
        timeout = int(timeout / 1000) * int(test.get('page_cycles', 1))
        # account for the pause the raptor webext runner takes after browser startup
        timeout += (int(self.post_startup_delay / 1000) + 3)

        # if geckoProfile enabled, give browser more time for profiling
        if self.config['gecko_profile'] is True:
            timeout += 5 * 60

        elapsed_time = 0
        while not self.control_server._finished:
            if self.config['enable_control_server_wait']:
                response = self.control_server_wait_get()
                if response == 'webext_status/__raptor_shutdownBrowser':
                    if self.config['memory_test']:
                        generate_android_memory_profile(self, test['name'])
                    self.control_server_wait_continue()
            time.sleep(1)
            # we only want to force browser-shutdown on timeout if not in debug mode;
            # in debug-mode we leave the browser running (require manual shutdown)
            if not self.debug_mode:
                elapsed_time += 1
                if elapsed_time > (timeout) - 5:  # stop 5 seconds early
                    self.log.info(
                        "application timed out after {} seconds".format(
                            timeout))
                    self.control_server.wait_for_quit()
                    break

    def process_results(self, test_names):
        # when running locally output results in build/raptor.json; when running
        # in production output to a local.json to be turned into tc job artifact
        if self.config.get('run_local', False):
            if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
                raptor_json_path = os.path.join(
                    os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing',
                    'mozharness', 'build', 'raptor.json')
            else:
                raptor_json_path = os.path.join(here, 'raptor.json')
        else:
            raptor_json_path = os.path.join(os.getcwd(), 'local.json')

        self.config['raptor_json_path'] = raptor_json_path
        return self.results_handler.summarize_and_output(
            self.config, test_names)

    def get_page_timeout_list(self):
        return self.results_handler.page_timeout_list

    def clean_up(self):
        if self.config['enable_control_server_wait']:
            self.control_server_wait_clear('all')

        self.control_server.stop()
        self.log.info("finished")

    def control_server_wait_set(self, state):
        response = requests.post("http://127.0.0.1:%s/" %
                                 self.control_server.port,
                                 json={
                                     "type": "wait-set",
                                     "data": state
                                 })
        return response.content

    def control_server_wait_timeout(self, timeout):
        response = requests.post("http://127.0.0.1:%s/" %
                                 self.control_server.port,
                                 json={
                                     "type": "wait-timeout",
                                     "data": timeout
                                 })
        return response.content

    def control_server_wait_get(self):
        response = requests.post("http://127.0.0.1:%s/" %
                                 self.control_server.port,
                                 json={
                                     "type": "wait-get",
                                     "data": ""
                                 })
        return response.content

    def control_server_wait_continue(self):
        response = requests.post("http://127.0.0.1:%s/" %
                                 self.control_server.port,
                                 json={
                                     "type": "wait-continue",
                                     "data": ""
                                 })
        return response.content

    def control_server_wait_clear(self, state):
        response = requests.post("http://127.0.0.1:%s/" %
                                 self.control_server.port,
                                 json={
                                     "type": "wait-clear",
                                     "data": state
                                 })
        return response.content