def run_test_setup(self, test): self.log.info("starting raptor test: %s" % test['name']) self.log.info("test settings: %s" % str(test)) self.log.info("raptor config: %s" % str(self.config)) if test.get('type') == "benchmark": self.serve_benchmark_source(test) gen_test_config( self.config['app'], test['name'], self.control_server.port, self.post_startup_delay, host=self.config['host'], b_port=self.benchmark_port, debug_mode=1 if self.debug_mode else 0, browser_cycle=test.get('browser_cycle', 1), ) self.install_raptor_webext() if test.get("preferences") is not None: self.set_browser_test_prefs(test['preferences']) # if 'alert_on' was provided in the test INI, add to our config for results/output self.config['subtest_alert_on'] = test.get('alert_on')
def run_test(self, test, timeout=None): self.log.info("starting raptor test: %s" % test['name']) self.log.info("test settings: %s" % str(test)) self.log.info("raptor config: %s" % str(self.config)) # benchmark-type tests require the benchmark test to be served out if test.get('type') == "benchmark": self.benchmark = Benchmark(self.config, test) benchmark_port = int(self.benchmark.port) else: benchmark_port = 0 gen_test_config(self.config['app'], test['name'], self.control_server.port, benchmark_port) # must intall raptor addon each time because we dynamically update some content raptor_webext = os.path.join(webext_dir, 'raptor') self.log.info("installing webext %s" % raptor_webext) self.profile.addons.install(raptor_webext) # on firefox we can get an addon id; chrome addon actually is just cmd line arg if self.config['app'] == "firefox": webext_id = self.profile.addons.addon_details(raptor_webext)['id'] # some tests require tools to playback the test pages if test.get('playback', None) is not None: self.get_playback_config(test) # startup the playback tool self.playback = get_playback(self.config) self.runner.start() proc = self.runner.process_handler self.output_handler.proc = proc self.control_server.browser_proc = proc try: self.runner.wait(timeout) finally: try: self.runner.check_for_crashes() except NotImplementedError: # not implemented for Chrome pass if self.playback is not None: self.playback.stop() # remove the raptor webext; as it must be reloaded with each subtest anyway # applies to firefox only; chrome the addon is actually just cmd line arg if self.config['app'] == "firefox": self.log.info("removing webext %s" % raptor_webext) self.profile.addons.remove_addon(webext_id) if self.runner.is_running(): self.log("Application timed out after {} seconds".format(timeout)) self.runner.stop()
def run_test(self, test, timeout=None): self.log.info("starting raptor test: %s" % test['name']) gen_test_config(self.config['app'], test['name'], self.control_server.port) # must intall raptor addon each time because we dynamically update some content raptor_webext = os.path.join(webext_dir, 'raptor') self.log.info("installing webext %s" % raptor_webext) self.profile.addons.install(raptor_webext) webext_id = self.profile.addons.addon_details(raptor_webext)['id'] # some tests require tools to playback the test pages if test.get('playback', None) is not None: self.get_playback_config(test) # startup the playback tool self.playback = get_playback(self.config) self.runner.start() proc = self.runner.process_handler self.output_handler.proc = proc self.control_server.browser_proc = proc try: self.runner.wait(timeout) finally: try: self.runner.check_for_crashes() except NotImplementedError: # not implemented for Chrome pass if self.playback is not None: self.playback.stop() # remove the raptor webext; as it must be reloaded with each subtest anyway self.log.info("removing webext %s" % raptor_webext) self.profile.addons.remove_addon(webext_id) if self.runner.is_running(): self.log("Application timed out after {} seconds".format(timeout)) self.runner.stop()
def run_test_setup(self, test): super(WebExtension, self).run_test_setup(test) LOG.info("starting web extension test: %s" % test["name"]) LOG.info("test settings: %s" % str(test)) LOG.info("web extension config: %s" % str(self.config)) if test.get("type") == "benchmark": self.serve_benchmark_source(test) gen_test_config( test["name"], self.control_server.port, self.post_startup_delay, host=self.config["host"], b_port=int(self.benchmark.port) if self.benchmark else 0, debug_mode=1 if self.debug_mode else 0, browser_cycle=test.get("browser_cycle", 1), ) self.install_raptor_webext()
def run_test(self, test, timeout=None): self.log.info("starting raptor test: %s" % test['name']) gen_test_config(self.config['app'], test['name'], self.control_server.port) self.profile.addons.install(os.path.join(webext_dir, 'raptor')) # some tests require tools to playback the test pages if test.get('playback', None) is not None: self.get_playback_config(test) # startup the playback tool self.playback = get_playback(self.config) self.runner.start() first_time = int(time.time()) * 1000 proc = self.runner.process_handler self.output_handler.proc = proc try: self.runner.wait(timeout) finally: try: self.runner.check_for_crashes() except NotImplementedError: # not implemented for Chrome pass if self.playback is not None: self.playback.stop() if self.runner.is_running(): self.log("Application timed out after {} seconds".format(timeout)) self.runner.stop() proc.output.append( "__startBeforeLaunchTimestamp%d__endBeforeLaunchTimestamp" % first_time) proc.output.append( "__startAfterTerminationTimestamp%d__endAfterTerminationTimestamp" % (int(time.time()) * 1000))
def run_test(self, test, timeout=None): self.log.info("starting raptor test: %s" % test['name']) self.log.info("test settings: %s" % str(test)) self.log.info("raptor config: %s" % str(self.config)) # benchmark-type tests require the benchmark test to be served out if test.get('type') == "benchmark": self.benchmark = Benchmark(self.config, test) benchmark_port = int(self.benchmark.port) else: benchmark_port = 0 gen_test_config(self.config['app'], test['name'], self.control_server.port, benchmark_port) # for android we must make the benchmarks server available to the device if self.config['app'] == "geckoview": self.log.info( "making the raptor benchmarks server port available to device") _tcp_port = "tcp:%s" % benchmark_port self.device.create_socket_connection('reverse', _tcp_port, _tcp_port) # must intall raptor addon each time because we dynamically update some content raptor_webext = os.path.join(webext_dir, 'raptor') self.log.info("installing webext %s" % raptor_webext) self.profile.addons.install(raptor_webext) # add test specific preferences if test.get("preferences", None) is not None: if self.config['app'] == "firefox": self.profile.set_preferences(json.loads(test['preferences'])) else: self.log.info("preferences were configured for the test, \ but we do not install them on non Firefox browsers." ) # on firefox we can get an addon id; chrome addon actually is just cmd line arg if self.config['app'] in ["firefox", "geckoview"]: webext_id = self.profile.addons.addon_details(raptor_webext)['id'] # some tests require tools to playback the test pages if test.get('playback', None) is not None: self.get_playback_config(test) # startup the playback tool self.playback = get_playback(self.config) # for geckoview we must copy the profile onto the device and set perms if self.config['app'] == "geckoview": if not self.device.is_app_installed(self.config['binary']): raise Exception('%s is not installed' % self.config['binary']) self.log.info("copying firefox profile onto the android device") self.device_profile = "/sdcard/raptor-profile" if self.device.is_dir(self.device_profile): self.device.rm(self.device_profile, recursive=True) self.device.mkdir(self.device_profile) self.device.push(self.profile.profile, self.device_profile) self.log.info("setting permisions to profile dir on the device") self.device.chmod(self.device_profile, recursive=True) # now start the geckoview app self.log.info("starting %s" % self.config['app']) extra_args = [ "-profile", self.device_profile, "--es", "env0", "LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6" ] try: # make sure the geckoview app is not running before # attempting to start. self.device.stop_application(self.config['binary']) self.device.launch_activity(self.config['binary'], "GeckoViewActivity", extra_args=extra_args, url='about:blank', fail_if_running=False) except Exception: self.log.error("Exception launching %s" % self.config['binary']) raise self.control_server.device = self.device self.control_server.app_name = self.config['binary'] else: # now start the desktop browser self.log.info("starting %s" % self.config['app']) self.runner.start() proc = self.runner.process_handler self.output_handler.proc = proc self.control_server.browser_proc = proc # set our cs flag to indicate we are running the browser/app self.control_server._finished = False # convert to seconds and account for page cycles timeout = int(timeout / 1000) * int(test['page_cycles']) try: elapsed_time = 0 while not self.control_server._finished: time.sleep(1) elapsed_time += 1 if elapsed_time > (timeout) - 5: # stop 5 seconds early self.log.info( "application timed out after {} seconds".format( timeout)) self.control_server.wait_for_quit() break finally: if self.config['app'] != "geckoview": try: self.runner.check_for_crashes() except NotImplementedError: # not implemented for Chrome pass # TODO: if on geckoview is there some cleanup here i.e. check for crashes? if self.playback is not None: self.playback.stop() # remove the raptor webext; as it must be reloaded with each subtest anyway # applies to firefox only; chrome the addon is actually just cmd line arg if self.config['app'] in ["firefox", "geckoview"]: self.log.info("removing webext %s" % raptor_webext) self.profile.addons.remove_addon(webext_id) if self.config['app'] != "geckoview": if self.runner.is_running(): self.runner.stop()
def run_test(self, test, timeout=None): self.log.info("starting raptor test: %s" % test['name']) self.log.info("test settings: %s" % str(test)) self.log.info("raptor config: %s" % str(self.config)) # benchmark-type tests require the benchmark test to be served out if test.get('type') == "benchmark": self.benchmark = Benchmark(self.config, test) benchmark_port = int(self.benchmark.port) # for android we must make the benchmarks server available to the device if self.config['app'] == "geckoview" and self.config['host'] \ in ('localhost', '127.0.0.1'): self.log.info( "making the raptor benchmarks server port available to device" ) _tcp_port = "tcp:%s" % benchmark_port self.device.create_socket_connection('reverse', _tcp_port, _tcp_port) else: benchmark_port = 0 gen_test_config(self.config['app'], test['name'], self.control_server.port, self.post_startup_delay, host=self.config['host'], b_port=benchmark_port, debug_mode=1 if self.debug_mode else 0) # must intall raptor addon each time because we dynamically update some content # note: for chrome the addon is just a list of paths that ultimately are added # to the chromium command line '--load-extension' argument raptor_webext = os.path.join(webext_dir, 'raptor') self.log.info("installing webext %s" % raptor_webext) self.profile.addons.install(raptor_webext) # add test specific preferences if test.get("preferences", None) is not None: if self.config['app'] == "firefox": self.profile.set_preferences(json.loads(test['preferences'])) else: self.log.info("preferences were configured for the test, \ but we do not install them on non Firefox browsers." ) # on firefox we can get an addon id; chrome addon actually is just cmd line arg if self.config['app'] in ["firefox", "geckoview"]: webext_id = self.profile.addons.addon_details(raptor_webext)['id'] # for android/geckoview, create a top-level raptor folder on the device # sdcard; if it already exists remove it so we start fresh each time if self.config['app'] == "geckoview": self.device_raptor_dir = "/sdcard/raptor" self.config['device_raptor_dir'] = self.device_raptor_dir if self.device.is_dir(self.device_raptor_dir): self.log.info("deleting existing device raptor dir: %s" % self.device_raptor_dir) self.device.rm(self.device_raptor_dir, recursive=True) self.log.info("creating raptor folder on sdcard: %s" % self.device_raptor_dir) self.device.mkdir(self.device_raptor_dir) self.device.chmod(self.device_raptor_dir, recursive=True) # some tests require tools to playback the test pages if test.get('playback', None) is not None: self.get_playback_config(test) # startup the playback tool self.playback = get_playback(self.config, self.device) # for android we must make the playback server available to the device if self.config['app'] == "geckoview" and self.config['host'] \ in ('localhost', '127.0.0.1'): self.log.info( "making the raptor playback server port available to device" ) _tcp_port = "tcp:8080" self.device.create_socket_connection('reverse', _tcp_port, _tcp_port) if self.config['app'] in ("geckoview", "firefox") and \ self.config['host'] not in ('localhost', '127.0.0.1'): # Must delete the proxy settings from the profile if running # the test with a host different from localhost. userjspath = os.path.join(self.profile.profile, 'user.js') with open(userjspath) as userjsfile: prefs = userjsfile.readlines() prefs = [pref for pref in prefs if 'network.proxy' not in pref] with open(userjspath, 'w') as userjsfile: userjsfile.writelines(prefs) # for geckoview/android pageload playback we can't use a policy to turn on the # proxy; we need to set prefs instead; note that the 'host' may be different # than '127.0.0.1' so we must set the prefs accordingly if self.config['app'] == "geckoview" and test.get('playback', None) is not None: self.log.info( "setting profile prefs to turn on the geckoview browser proxy") no_proxies_on = "localhost, 127.0.0.1, %s" % self.config['host'] proxy_prefs = {} proxy_prefs["network.proxy.type"] = 1 proxy_prefs["network.proxy.http"] = self.config['host'] proxy_prefs["network.proxy.http_port"] = 8080 proxy_prefs["network.proxy.ssl"] = self.config['host'] proxy_prefs["network.proxy.ssl_port"] = 8080 proxy_prefs["network.proxy.no_proxies_on"] = no_proxies_on self.profile.set_preferences(proxy_prefs) # now some final settings, and then startup of the browser under test if self.config['app'] == "geckoview": # for android/geckoview we must copy the profile onto the device and set perms if not self.device.is_app_installed(self.config['binary']): raise Exception('%s is not installed' % self.config['binary']) self.device_profile = os.path.join(self.device_raptor_dir, "profile") if self.device.is_dir(self.device_profile): self.log.info("deleting existing device profile folder: %s" % self.device_profile) self.device.rm(self.device_profile, recursive=True) self.log.info("creating profile folder on device: %s" % self.device_profile) self.device.mkdir(self.device_profile) self.log.info("copying firefox profile onto the device") self.log.info("note: the profile folder being copied is: %s" % self.profile.profile) self.log.info( 'the adb push cmd copies that profile dir to a new temp dir before copy' ) self.device.push(self.profile.profile, self.device_profile) self.device.chmod(self.device_profile, recursive=True) # now start the geckoview app self.log.info("starting %s" % self.config['app']) extra_args = [ "-profile", self.device_profile, "--es", "env0", "LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6" ] try: # make sure the geckoview app is not running before # attempting to start. self.device.stop_application(self.config['binary']) self.device.launch_activity(self.config['binary'], "GeckoViewActivity", extra_args=extra_args, url='about:blank', e10s=True, fail_if_running=False) except Exception: self.log.error("Exception launching %s" % self.config['binary']) raise self.control_server.device = self.device self.control_server.app_name = self.config['binary'] else: # For Firefox we need to set # MOZ_DISABLE_NONLOCAL_CONNECTIONS=1 env var before # startup when testing release builds from mozilla-beta or # mozilla-release. This is because of restrictions on # release builds that require webextensions to be signed # unless MOZ_DISABLE_NONLOCAL_CONNECTIONS is set to '1'. if self.config['app'] == "firefox" and self.config[ 'is_release_build']: self.log.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=1") os.environ['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = "1" # if running debug-mode, tell Firefox to open the browser console on startup # for google chrome, open the devtools on the raptor test tab if self.debug_mode: if self.config['app'] == "firefox": self.runner.cmdargs.extend(['-jsconsole']) if self.config['app'] == "chrome": self.runner.cmdargs.extend( ['--auto-open-devtools-for-tabs']) # now start the desktop browser self.log.info("starting %s" % self.config['app']) # if running a pageload test on google chrome, add the cmd line options # to turn on the proxy and ignore security certificate errors # if using host localhost, 127.0.0.1. if self.config['app'] == "chrome" and test.get('playback', None) is not None: chrome_args = [ '--proxy-server="http=127.0.0.1:8080;' + 'https=127.0.0.1:8080;ssl=127.0.0.1:8080"', '--ignore-certificate-errors' ] if self.config['host'] not in ('localhost', '127.0.0.1'): chrome_args[0] = chrome_args[0].replace( '127.0.0.1', self.config['host']) if ' '.join(chrome_args) not in ' '.join(self.runner.cmdargs): self.runner.cmdargs.extend(chrome_args) self.runner.start() proc = self.runner.process_handler self.output_handler.proc = proc self.control_server.browser_proc = proc # pageload tests need to be able to access non-local connections via mitmproxy if self.config['app'] == "firefox" and self.config['is_release_build'] and \ test.get('playback', None) is not None: self.log.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=0") os.environ['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = "0" # if geckoProfile is enabled, initialize it if self.config['gecko_profile'] is True: self._init_gecko_profiling(test) # tell the control server the gecko_profile dir; the control server will # receive the actual gecko profiles from the web ext and will write them # to disk; then profiles are picked up by gecko_profile.symbolicate self.control_server.gecko_profile_dir = self.gecko_profiler.gecko_profile_dir # set our cs flag to indicate we are running the browser/app self.control_server._finished = False # convert to seconds and account for page cycles timeout = int(timeout / 1000) * int(test['page_cycles']) # account for the pause the raptor webext runner takes after browser startup timeout += int(self.post_startup_delay / 1000) # if geckoProfile enabled, give browser more time for profiling if self.config['gecko_profile'] is True: timeout += 5 * 60 try: elapsed_time = 0 while not self.control_server._finished: time.sleep(1) # we only want to force browser-shutdown on timeout if not in debug mode; # in debug-mode we leave the browser running (require manual shutdown) if not self.debug_mode: elapsed_time += 1 if elapsed_time > (timeout) - 5: # stop 5 seconds early self.log.info( "application timed out after {} seconds".format( timeout)) self.control_server.wait_for_quit() break finally: if self.config['app'] != "geckoview": try: self.runner.check_for_crashes() except NotImplementedError: # not implemented for Chrome pass # TODO: if on geckoview is there some cleanup here i.e. check for crashes? if self.playback is not None: self.playback.stop() # remove the raptor webext; as it must be reloaded with each subtest anyway self.log.info("removing webext %s" % raptor_webext) if self.config['app'] in ["firefox", "geckoview"]: self.profile.addons.remove_addon(webext_id) # for chrome the addon is just a list (appended to cmd line) if self.config['app'] in ["chrome", "chrome-android"]: self.profile.addons.remove(raptor_webext) # gecko profiling symbolication if self.config['gecko_profile'] is True: self.gecko_profiler.symbolicate() # clean up the temp gecko profiling folders self.log.info("cleaning up after gecko profiling") self.gecko_profiler.clean() # browser should be closed by now but this is a backup-shutdown (if not in debug-mode) if not self.debug_mode: if self.config['app'] != "geckoview": if self.runner.is_running(): self.runner.stop() # TODO the geckoview app should have been shutdown by this point by the # control server, but we can double-check here to make sure else: # in debug mode, and running locally, leave the browser running if self.config['run_local']: self.log.info( "* debug-mode enabled - please shutdown the browser manually..." ) self.runner.wait(timeout=None)
def run_test(self, test, timeout=None): self.log.info("starting raptor test: %s" % test['name']) self.log.info("test settings: %s" % str(test)) self.log.info("raptor config: %s" % str(self.config)) # benchmark-type tests require the benchmark test to be served out if test.get('type') == "benchmark": self.benchmark = Benchmark(self.config, test) benchmark_port = int(self.benchmark.port) else: benchmark_port = 0 gen_test_config(self.config['app'], test['name'], self.control_server.port, self.post_startup_delay, benchmark_port) # for android we must make the benchmarks server available to the device if self.config['app'] == "geckoview": self.log.info( "making the raptor benchmarks server port available to device") _tcp_port = "tcp:%s" % benchmark_port self.device.create_socket_connection('reverse', _tcp_port, _tcp_port) # must intall raptor addon each time because we dynamically update some content # note: for chrome the addon is just a list of paths that ultimately are added # to the chromium command line '--load-extension' argument raptor_webext = os.path.join(webext_dir, 'raptor') self.log.info("installing webext %s" % raptor_webext) self.profile.addons.install(raptor_webext) # add test specific preferences if test.get("preferences", None) is not None: if self.config['app'] == "firefox": self.profile.set_preferences(json.loads(test['preferences'])) else: self.log.info("preferences were configured for the test, \ but we do not install them on non Firefox browsers." ) # on firefox we can get an addon id; chrome addon actually is just cmd line arg if self.config['app'] in ["firefox", "geckoview"]: webext_id = self.profile.addons.addon_details(raptor_webext)['id'] # some tests require tools to playback the test pages if test.get('playback', None) is not None: self.get_playback_config(test) # startup the playback tool self.playback = get_playback(self.config) # for geckoview we must copy the profile onto the device and set perms if self.config['app'] == "geckoview": if not self.device.is_app_installed(self.config['binary']): raise Exception('%s is not installed' % self.config['binary']) self.log.info("copying firefox profile onto the android device") self.device_profile = "/sdcard/raptor-profile" if self.device.is_dir(self.device_profile): self.device.rm(self.device_profile, recursive=True) self.device.mkdir(self.device_profile) self.device.push(self.profile.profile, self.device_profile) self.log.info("setting permisions to profile dir on the device") self.device.chmod(self.device_profile, recursive=True) # now start the geckoview app self.log.info("starting %s" % self.config['app']) extra_args = [ "-profile", self.device_profile, "--es", "env0", "LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6" ] try: # make sure the geckoview app is not running before # attempting to start. self.device.stop_application(self.config['binary']) self.device.launch_activity(self.config['binary'], "GeckoViewActivity", extra_args=extra_args, url='about:blank', fail_if_running=False) except Exception: self.log.error("Exception launching %s" % self.config['binary']) raise self.control_server.device = self.device self.control_server.app_name = self.config['binary'] else: # now start the desktop browser self.log.info("starting %s" % self.config['app']) # if running a pageload test on google chrome, add the cmd line options # to turn on the proxy and ignore security certificate errors if self.config['app'] == "chrome" and test.get('playback', None) is not None: chrome_args = [ '--proxy-server="http=127.0.0.1:8080;https=127.0.0.1:8080;ssl=127.0.0.1:8080"', '--ignore-certificate-errors' ] self.runner.cmdargs.extend(chrome_args) self.runner.start() proc = self.runner.process_handler self.output_handler.proc = proc self.control_server.browser_proc = proc # if geckoProfile is enabled, initialize it if self.config['gecko_profile'] is True: self._init_gecko_profiling(test) # tell the control server the gecko_profile dir; the control server will # receive the actual gecko profiles from the web ext and will write them # to disk; then profiles are picked up by gecko_profile.symbolicate self.control_server.gecko_profile_dir = self.gecko_profiler.gecko_profile_dir # set our cs flag to indicate we are running the browser/app self.control_server._finished = False # convert to seconds and account for page cycles timeout = int(timeout / 1000) * int(test['page_cycles']) # account for the pause the raptor webext runner takes after browser startup timeout += int(self.post_startup_delay / 1000) # if geckoProfile enabled, give browser more time for profiling if self.config['gecko_profile'] is True: timeout += 5 * 60 try: elapsed_time = 0 while not self.control_server._finished: time.sleep(1) elapsed_time += 1 if elapsed_time > (timeout) - 5: # stop 5 seconds early self.log.info( "application timed out after {} seconds".format( timeout)) self.control_server.wait_for_quit() break finally: if self.config['app'] != "geckoview": try: self.runner.check_for_crashes() except NotImplementedError: # not implemented for Chrome pass # TODO: if on geckoview is there some cleanup here i.e. check for crashes? if self.playback is not None: self.playback.stop() # remove the raptor webext; as it must be reloaded with each subtest anyway self.log.info("removing webext %s" % raptor_webext) if self.config['app'] in ["firefox", "geckoview"]: self.profile.addons.remove_addon(webext_id) # for chrome the addon is just a list (appended to cmd line) if self.config['app'] in ["chrome", "chrome-android"]: self.profile.addons.remove(raptor_webext) # gecko profiling symbolication if self.config['gecko_profile'] is True: self.gecko_profiler.symbolicate() # clean up the temp gecko profiling folders self.log.info("cleaning up after gecko profiling") self.gecko_profiler.clean() if self.config['app'] != "geckoview": if self.runner.is_running(): self.runner.stop()