def start_control_server(self):
self.control_server = RaptorControlServer(self.results_handler, self.debug_mode)
self.control_server.user_profile = self.profile
self.control_server.start()
if self.config["enable_control_server_wait"]:
self.control_server_wait_set("webext_shutdownBrowser")
def start_control_server(self):
self.control_server = RaptorControlServer(self.results_handler,
self.debug_mode)
self.control_server.start()
if self.config['enable_control_server_wait']:
self.control_server_wait_set(
'webext_status/__raptor_shutdownBrowser')
def start_control_server(self):
self.control_server = RaptorControlServer(self.results_handler)
self.control_server.start()
# for android we must make the control server available to the device
if self.config['app'] == "geckoview":
self.log.info("making the raptor control server port available to device")
_tcp_port = "tcp:%s" % self.control_server.port
self.device.create_socket_connection('reverse', _tcp_port, _tcp_port)
def start_control_server(self):
self.control_server = RaptorControlServer(self.results_handler, self.debug_mode)
self.control_server.start()
# for android we must make the control server available to the device
if self.config['app'] in self.firefox_android_apps and \
self.config['host'] in ('localhost', '127.0.0.1'):
self.log.info("making the raptor control server port available to device")
_tcp_port = "tcp:%s" % self.control_server.port
self.device.create_socket_connection('reverse', _tcp_port, _tcp_port)
def start_control_server(self):
self.control_server = RaptorControlServer(self.results_handler)
self.control_server.start()
# for android we must make the control server available to the device
if self.config['app'] == "geckoview":
self.log.info("making the raptor control server port available to device")
_tcp_port = "tcp:%s" % self.control_server.port
_cmd = ["reverse", _tcp_port, _tcp_port]
self.device.command_output(_cmd)
class Raptor(object):
"""Container class for Raptor"""
def __init__(self, app, binary, run_local=False, obj_path=None):
self.config = {}
self.config['app'] = app
self.config['binary'] = binary
self.config['platform'] = mozinfo.os
self.config['processor'] = mozinfo.processor
self.config['run_local'] = run_local
self.config['obj_path'] = obj_path
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
self.log = get_default_logger(component='raptor-main')
self.control_server = None
self.playback = None
self.benchmark = None
# Create the profile; for geckoview we want a firefox profile type
if self.config['app'] == 'geckoview':
self.profile = create_profile('firefox')
else:
self.profile = create_profile(self.config['app'])
# Merge in base profiles
with open(os.path.join(self.profile_data_dir, 'profiles.json'),
'r') as fh:
base_profiles = json.load(fh)['raptor']
for name in base_profiles:
path = os.path.join(self.profile_data_dir, name)
self.log.info("Merging profile: {}".format(path))
self.profile.merge(path)
# create results holder
self.results_handler = RaptorResultsHandler()
# when testing desktop browsers we use mozrunner to start the browser; when
# testing on android (i.e. geckoview) we use mozdevice to control the device app
if self.config['app'] == "geckoview":
# create the android device handler; it gets initiated and sets up adb etc
self.log.info("creating android device handler using mozdevice")
self.device = ADBAndroid(verbose=True)
self.device.clear_logcat()
else:
# create the desktop browser runner
self.log.info("creating browser runner using mozrunner")
self.output_handler = OutputHandler()
process_args = {
'processOutputLine': [self.output_handler],
}
runner_cls = runners[app]
self.runner = runner_cls(binary,
profile=self.profile,
process_args=process_args)
self.log.info("raptor config: %s" % str(self.config))
@property
def profile_data_dir(self):
if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
'testing', 'profiles')
if build:
return os.path.join(build.topsrcdir, 'testing', 'profiles')
return os.path.join(here, 'profile_data')
def start_control_server(self):
self.control_server = RaptorControlServer(self.results_handler)
self.control_server.start()
# for android we must make the control server available to the device
if self.config['app'] == "geckoview":
self.log.info(
"making the raptor control server port available to device")
_tcp_port = "tcp:%s" % self.control_server.port
self.device.create_socket_connection('reverse', _tcp_port,
_tcp_port)
def get_playback_config(self, test):
self.config['playback_tool'] = test.get('playback')
self.log.info("test uses playback tool: %s " %
self.config['playback_tool'])
self.config['playback_binary_manifest'] = test.get(
'playback_binary_manifest', None)
_key = 'playback_binary_zip_%s' % self.config['platform']
self.config['playback_binary_zip'] = test.get(_key, None)
self.config['playback_pageset_manifest'] = test.get(
'playback_pageset_manifest', None)
_key = 'playback_pageset_zip_%s' % self.config['platform']
self.config['playback_pageset_zip'] = test.get(_key, None)
self.config['playback_recordings'] = test.get('playback_recordings',
None)
self.config['python3_win_manifest'] = test.get('python3_win_manifest',
None)
def run_test(self, test, timeout=None):
self.log.info("starting raptor test: %s" % test['name'])
self.log.info("test settings: %s" % str(test))
self.log.info("raptor config: %s" % str(self.config))
# benchmark-type tests require the benchmark test to be served out
if test.get('type') == "benchmark":
self.benchmark = Benchmark(self.config, test)
benchmark_port = int(self.benchmark.port)
else:
benchmark_port = 0
gen_test_config(self.config['app'], test['name'],
self.control_server.port, benchmark_port)
# for android we must make the benchmarks server available to the device
if self.config['app'] == "geckoview":
self.log.info(
"making the raptor benchmarks server port available to device")
_tcp_port = "tcp:%s" % benchmark_port
self.device.create_socket_connection('reverse', _tcp_port,
_tcp_port)
# must intall raptor addon each time because we dynamically update some content
raptor_webext = os.path.join(webext_dir, 'raptor')
self.log.info("installing webext %s" % raptor_webext)
self.profile.addons.install(raptor_webext)
# add test specific preferences
if test.get("preferences", None) is not None:
if self.config['app'] == "firefox":
self.profile.set_preferences(json.loads(test['preferences']))
else:
self.log.info("preferences were configured for the test, \
but we do not install them on non Firefox browsers."
)
# on firefox we can get an addon id; chrome addon actually is just cmd line arg
if self.config['app'] in ["firefox", "geckoview"]:
webext_id = self.profile.addons.addon_details(raptor_webext)['id']
# some tests require tools to playback the test pages
if test.get('playback', None) is not None:
self.get_playback_config(test)
# startup the playback tool
self.playback = get_playback(self.config)
# for geckoview we must copy the profile onto the device and set perms
if self.config['app'] == "geckoview":
if not self.device.is_app_installed(self.config['binary']):
raise Exception('%s is not installed' % self.config['binary'])
self.log.info("copying firefox profile onto the android device")
self.device_profile = "/sdcard/raptor-profile"
if self.device.is_dir(self.device_profile):
self.device.rm(self.device_profile, recursive=True)
self.device.mkdir(self.device_profile)
self.device.push(self.profile.profile, self.device_profile)
self.log.info("setting permisions to profile dir on the device")
self.device.chmod(self.device_profile, recursive=True)
# now start the geckoview app
self.log.info("starting %s" % self.config['app'])
extra_args = [
"-profile", self.device_profile, "--es", "env0",
"LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6"
]
try:
# make sure the geckoview app is not running before
# attempting to start.
self.device.stop_application(self.config['binary'])
self.device.launch_activity(self.config['binary'],
"GeckoViewActivity",
extra_args=extra_args,
url='about:blank',
fail_if_running=False)
except Exception:
self.log.error("Exception launching %s" %
self.config['binary'])
raise
self.control_server.device = self.device
self.control_server.app_name = self.config['binary']
else:
# now start the desktop browser
self.log.info("starting %s" % self.config['app'])
self.runner.start()
proc = self.runner.process_handler
self.output_handler.proc = proc
self.control_server.browser_proc = proc
# set our cs flag to indicate we are running the browser/app
self.control_server._finished = False
# convert to seconds and account for page cycles
timeout = int(timeout / 1000) * int(test['page_cycles'])
try:
elapsed_time = 0
while not self.control_server._finished:
time.sleep(1)
elapsed_time += 1
if elapsed_time > (timeout) - 5: # stop 5 seconds early
self.log.info(
"application timed out after {} seconds".format(
timeout))
self.control_server.wait_for_quit()
break
finally:
if self.config['app'] != "geckoview":
try:
self.runner.check_for_crashes()
except NotImplementedError: # not implemented for Chrome
pass
# TODO: if on geckoview is there some cleanup here i.e. check for crashes?
if self.playback is not None:
self.playback.stop()
# remove the raptor webext; as it must be reloaded with each subtest anyway
# applies to firefox only; chrome the addon is actually just cmd line arg
if self.config['app'] in ["firefox", "geckoview"]:
self.log.info("removing webext %s" % raptor_webext)
self.profile.addons.remove_addon(webext_id)
if self.config['app'] != "geckoview":
if self.runner.is_running():
self.runner.stop()
# TODO the geckoview app should have been shutdown by this point by the
# control server, but we can double-check here to make sure
def process_results(self):
# when running locally output results in build/raptor.json; when running
# in production output to a local.json to be turned into tc job artifact
if self.config.get('run_local', False):
if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
raptor_json_path = os.path.join(
os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing',
'mozharness', 'build', 'raptor.json')
else:
raptor_json_path = os.path.join(here, 'raptor.json')
else:
raptor_json_path = os.path.join(os.getcwd(), 'local.json')
self.config['raptor_json_path'] = raptor_json_path
return self.results_handler.summarize_and_output(self.config)
def clean_up(self):
self.control_server.stop()
if self.config['app'] != "geckoview":
self.runner.stop()
elif self.config['app'] == 'geckoview':
self.log.info('removing reverse socket connections')
self.device.remove_socket_connections('reverse')
else:
pass
self.log.info("finished")
class WebExtension(Perftest):
"""Container class for WebExtension."""
def __init__(self, *args, **kwargs):
self.raptor_webext = None
self.control_server = None
self.cpu_profiler = None
super(WebExtension, self).__init__(*args, **kwargs)
# set up the results handler
self.results_handler = RaptorResultsHandler(**self.config)
browser_name, browser_version = self.get_browser_meta()
self.results_handler.add_browser_meta(self.config["app"], browser_version)
self.start_control_server()
def run_test_setup(self, test):
super(WebExtension, self).run_test_setup(test)
LOG.info("starting web extension test: %s" % test["name"])
LOG.info("test settings: %s" % str(test))
LOG.info("web extension config: %s" % str(self.config))
if test.get("type") == "benchmark":
self.serve_benchmark_source(test)
gen_test_config(
test["name"],
self.control_server.port,
self.post_startup_delay,
host=self.config["host"],
b_port=int(self.benchmark.port) if self.benchmark else 0,
debug_mode=1 if self.debug_mode else 0,
browser_cycle=test.get("browser_cycle", 1),
)
self.install_raptor_webext()
def wait_for_test_finish(self, test, timeout, process_exists_callback=None):
# this is a 'back-stop' i.e. if for some reason Raptor doesn't finish for some
# serious problem; i.e. the test was unable to send a 'page-timeout' to the control
# server, etc. Therefore since this is a 'back-stop' we want to be generous here;
# we don't want this timeout occurring unless abosultely necessary
# convert timeout to seconds and account for page cycles
# pylint --py3k W1619
timeout = int(timeout / 1000) * int(test.get("page_cycles", 1))
# account for the pause the raptor webext runner takes after browser startup
# and the time an exception is propagated through the framework
# pylint --py3k W1619
timeout += int(self.post_startup_delay / 1000) + 10
# for page-load tests we don't start the page-timeout timer until the pageload.js content
# is successfully injected and invoked; which differs per site being tested; therefore we
# need to be generous here - let's add 10 seconds extra per page-cycle
if test.get("type") == "pageload":
timeout += 10 * int(test.get("page_cycles", 1))
# if geckoProfile enabled, give browser more time for profiling
if self.config["gecko_profile"] is True:
timeout += 5 * 60
# we also need to give time for results processing, not just page/browser cycles!
timeout += 60
# stop 5 seconds early
end_time = time.time() + timeout - 5
while not self.control_server._finished:
# Ignore check if the control server shutdown the app
if not self.control_server._is_shutting_down:
# If the application is no longer running immediately bail out
if callable(process_exists_callback) and not process_exists_callback():
raise RuntimeError("Process has been unexpectedly closed")
if self.config["enable_control_server_wait"]:
response = self.control_server_wait_get()
if response == "webext_shutdownBrowser":
if self.config["memory_test"]:
generate_android_memory_profile(self, test["name"])
if self.cpu_profiler:
self.cpu_profiler.generate_android_cpu_profile(test["name"])
self.control_server_wait_continue()
# Sleep for a moment to not check the process too often
time.sleep(1)
# we only want to force browser-shutdown on timeout if not in debug mode;
# in debug-mode we leave the browser running (require manual shutdown)
if not self.debug_mode and end_time < time.time():
self.control_server.wait_for_quit()
if not self.control_server.is_webextension_loaded:
raise RuntimeError("Connection to Raptor webextension failed!")
raise RuntimeError(
"Test failed to finish. "
"Application timed out after {} seconds".format(timeout)
)
if self.control_server._runtime_error:
raise RuntimeError(
"Failed to run {}: {}\nStack:\n{}".format(
test["name"],
self.control_server._runtime_error["error"],
self.control_server._runtime_error["stack"],
)
)
def run_test_teardown(self, test):
super(WebExtension, self).run_test_teardown(test)
if self.playback is not None:
self.playback.stop()
self.playback = None
self.remove_raptor_webext()
def set_browser_test_prefs(self, raw_prefs):
# add test specific preferences
LOG.info("setting test-specific Firefox preferences")
self.profile.set_preferences(json.loads(raw_prefs))
def build_browser_profile(self):
super(WebExtension, self).build_browser_profile()
if self.control_server:
# The control server and the browser profile are not well factored
# at this time, so the start-up process overlaps. Accommodate.
self.control_server.user_profile = self.profile
def start_control_server(self):
self.control_server = RaptorControlServer(self.results_handler, self.debug_mode)
self.control_server.user_profile = self.profile
self.control_server.start()
if self.config["enable_control_server_wait"]:
self.control_server_wait_set("webext_shutdownBrowser")
def serve_benchmark_source(self, test):
# benchmark-type tests require the benchmark test to be served out
self.benchmark = Benchmark(self.config, test)
def install_raptor_webext(self):
# must intall raptor addon each time because we dynamically update some content
# the webext is installed into the browser profile
# note: for chrome the addon is just a list of paths that ultimately are added
# to the chromium command line '--load-extension' argument
self.raptor_webext = os.path.join(webext_dir, "raptor")
LOG.info("installing webext %s" % self.raptor_webext)
self.profile.addons.install(self.raptor_webext)
# on firefox we can get an addon id; chrome addon actually is just cmd line arg
try:
self.webext_id = self.profile.addons.addon_details(self.raptor_webext)["id"]
except AttributeError:
self.webext_id = None
self.control_server.startup_handler(False)
def remove_raptor_webext(self):
# remove the raptor webext; as it must be reloaded with each subtest anyway
if not self.raptor_webext:
LOG.info("raptor webext not installed - not attempting removal")
return
LOG.info("removing webext %s" % self.raptor_webext)
if self.config["app"] in ["firefox", "geckoview", "refbrow", "fenix"]:
self.profile.addons.remove_addon(self.webext_id)
# for chrome the addon is just a list (appended to cmd line)
chrome_apps = CHROMIUM_DISTROS + ["chrome-android", "chromium-android"]
if self.config["app"] in chrome_apps:
self.profile.addons.remove(self.raptor_webext)
self.raptor_webext = None
def clean_up(self):
super(WebExtension, self).clean_up()
if self.config["enable_control_server_wait"]:
self.control_server_wait_clear("all")
self.control_server.stop()
LOG.info("finished")
def control_server_wait_set(self, state):
response = requests.post(
"http://127.0.0.1:%s/" % self.control_server.port,
json={"type": "wait-set", "data": state},
)
return response.text
def control_server_wait_timeout(self, timeout):
response = requests.post(
"http://127.0.0.1:%s/" % self.control_server.port,
json={"type": "wait-timeout", "data": timeout},
)
return response.text
def control_server_wait_get(self):
response = requests.post(
"http://127.0.0.1:%s/" % self.control_server.port,
json={"type": "wait-get", "data": ""},
)
return response.text
def control_server_wait_continue(self):
response = requests.post(
"http://127.0.0.1:%s/" % self.control_server.port,
json={"type": "wait-continue", "data": ""},
)
return response.text
def control_server_wait_clear(self, state):
response = requests.post(
"http://127.0.0.1:%s/" % self.control_server.port,
json={"type": "wait-clear", "data": state},
)
return response.text
class Raptor(object):
"""Container class for Raptor"""
def __init__(self, app, binary, run_local=False, obj_path=None,
gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None,
symbols_path=None, host=None, power_test=False, is_release_build=False,
debug_mode=False, activity=None):
# Override the magic --host HOST_IP with the value of the environment variable.
if host == 'HOST_IP':
host = os.environ['HOST_IP']
self.config = {}
self.config['app'] = app
self.config['binary'] = binary
self.config['platform'] = mozinfo.os
self.config['processor'] = mozinfo.processor
self.config['run_local'] = run_local
self.config['obj_path'] = obj_path
self.config['gecko_profile'] = gecko_profile
self.config['gecko_profile_interval'] = gecko_profile_interval
self.config['gecko_profile_entries'] = gecko_profile_entries
self.config['symbols_path'] = symbols_path
self.config['host'] = host
self.config['power_test'] = power_test
self.config['is_release_build'] = is_release_build
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
self.log = get_default_logger(component='raptor-main')
self.control_server = None
self.playback = None
self.benchmark = None
self.benchmark_port = 0
self.gecko_profiler = None
self.post_startup_delay = 30000
self.device = None
self.profile_class = app
self.firefox_android_apps = ['fennec', 'geckoview', 'refbrow', 'fenix']
# debug mode is currently only supported when running locally
self.debug_mode = debug_mode if self.config['run_local'] else False
# if running debug-mode reduce the pause after browser startup
if self.debug_mode:
self.post_startup_delay = 3000
self.log.info("debug-mode enabled, reducing post-browser startup pause to %d ms"
% self.post_startup_delay)
self.log.info("main raptor init, config is: %s" % str(self.config))
# create results holder
self.results_handler = RaptorResultsHandler()
@property
def profile_data_dir(self):
if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'profiles')
if build:
return os.path.join(build.topsrcdir, 'testing', 'profiles')
return os.path.join(here, 'profile_data')
def run_test_setup(self, test):
self.log.info("starting raptor test: %s" % test['name'])
self.log.info("test settings: %s" % str(test))
self.log.info("raptor config: %s" % str(self.config))
if test.get('type') == "benchmark":
self.serve_benchmark_source(test)
gen_test_config(self.config['app'],
test['name'],
self.control_server.port,
self.post_startup_delay,
host=self.config['host'],
b_port=self.benchmark_port,
debug_mode=1 if self.debug_mode else 0)
self.install_raptor_webext()
if test.get("preferences", None) is not None:
self.set_browser_test_prefs(test['preferences'])
# if 'alert_on' was provided in the test INI, add to our config for results/output
self.config['subtest_alert_on'] = test.get('alert_on', None)
def set_browser_test_prefs(self, raw_prefs):
# add test specific preferences
if self.config['app'] == "firefox":
self.log.info("setting test-specific browser preferences")
self.profile.set_preferences(json.loads(raw_prefs))
else:
self.log.info("preferences were configured for the test, however \
we currently do not install them on non Firefox browsers.")
def run_test_teardown(self):
self.check_for_crashes()
if self.playback is not None:
self.playback.stop()
self.remove_raptor_webext()
# gecko profiling symbolication
if self.config['gecko_profile'] is True:
self.gecko_profiler.symbolicate()
# clean up the temp gecko profiling folders
self.log.info("cleaning up after gecko profiling")
self.gecko_profiler.clean()
def create_browser_profile(self):
self.profile = create_profile(self.profile_class)
# Merge in base profiles
with open(os.path.join(self.profile_data_dir, 'profiles.json'), 'r') as fh:
base_profiles = json.load(fh)['raptor']
for name in base_profiles:
path = os.path.join(self.profile_data_dir, name)
self.log.info("Merging profile: {}".format(path))
self.profile.merge(path)
# add profile dir to our config
self.config['local_profile_dir'] = self.profile.profile
def start_control_server(self):
self.control_server = RaptorControlServer(self.results_handler, self.debug_mode)
self.control_server.start()
# for android we must make the control server available to the device
if self.config['app'] in self.firefox_android_apps and \
self.config['host'] in ('localhost', '127.0.0.1'):
self.log.info("making the raptor control server port available to device")
_tcp_port = "tcp:%s" % self.control_server.port
self.device.create_socket_connection('reverse', _tcp_port, _tcp_port)
def get_playback_config(self, test):
self.config['playback_tool'] = test.get('playback')
self.log.info("test uses playback tool: %s " % self.config['playback_tool'])
self.config['playback_binary_manifest'] = test.get('playback_binary_manifest', None)
_key = 'playback_binary_zip_%s' % self.config['platform']
self.config['playback_binary_zip'] = test.get(_key, None)
self.config['playback_pageset_manifest'] = test.get('playback_pageset_manifest', None)
_key = 'playback_pageset_zip_%s' % self.config['platform']
self.config['playback_pageset_zip'] = test.get(_key, None)
self.config['playback_recordings'] = test.get('playback_recordings', None)
playback_dir = os.path.join(here, 'playback')
for key in ('playback_pageset_manifest', 'playback_pageset_zip'):
if self.config.get(key) is None:
continue
self.config[key] = os.path.join(playback_dir, self.config[key])
self.config['custom_script'] = os.path.join(playback_dir, 'alternate-server-replay.py')
def serve_benchmark_source(self, test):
# benchmark-type tests require the benchmark test to be served out
self.benchmark = Benchmark(self.config, test)
self.benchmark_port = int(self.benchmark.port)
# for android we must make the benchmarks server available to the device
if self.config['app'] in self.firefox_android_apps and \
self.config['host'] in ('localhost', '127.0.0.1'):
self.log.info("making the raptor benchmarks server port available to device")
_tcp_port = "tcp:%s" % self.benchmark_port
self.device.create_socket_connection('reverse', _tcp_port, _tcp_port)
def install_raptor_webext(self):
# must intall raptor addon each time because we dynamically update some content
# the webext is installed into the browser profile
# note: for chrome the addon is just a list of paths that ultimately are added
# to the chromium command line '--load-extension' argument
self.raptor_webext = os.path.join(webext_dir, 'raptor')
self.log.info("installing webext %s" % self.raptor_webext)
self.profile.addons.install(self.raptor_webext)
# on firefox we can get an addon id; chrome addon actually is just cmd line arg
try:
self.webext_id = self.profile.addons.addon_details(self.raptor_webext)['id']
except AttributeError:
self.webext_id = None
def remove_raptor_webext(self):
# remove the raptor webext; as it must be reloaded with each subtest anyway
self.log.info("removing webext %s" % self.raptor_webext)
if self.config['app'] in ['firefox', 'geckoview', 'fennec', 'refbrow', 'fenix']:
self.profile.addons.remove_addon(self.webext_id)
# for chrome the addon is just a list (appended to cmd line)
if self.config['app'] in ["chrome", "chrome-android"]:
self.profile.addons.remove(self.raptor_webext)
def set_test_browser_prefs(self, test_prefs):
# if the test has any specific browser prefs specified, set those in browser profiile
if self.config['app'] == "firefox":
self.profile.set_preferences(json.loads(test_prefs))
else:
self.log.info("preferences were configured for the test, \
but we do not install them on non Firefox browsers.")
def start_playback(self, test):
# startup the playback tool
self.get_playback_config(test)
self.playback = get_playback(self.config, self.device)
# for android we must make the playback server available to the device
if self.config['app'] in self.firefox_android_apps and self.config['host'] \
in ('localhost', '127.0.0.1'):
self.log.info("making the raptor playback server port available to device")
_tcp_port = "tcp:8080"
self.device.create_socket_connection('reverse', _tcp_port, _tcp_port)
def delete_proxy_settings_from_profile(self):
# Must delete the proxy settings from the profile if running
# the test with a host different from localhost.
userjspath = os.path.join(self.profile.profile, 'user.js')
with open(userjspath) as userjsfile:
prefs = userjsfile.readlines()
prefs = [pref for pref in prefs if 'network.proxy' not in pref]
with open(userjspath, 'w') as userjsfile:
userjsfile.writelines(prefs)
def _init_gecko_profiling(self, test):
self.log.info("initializing gecko profiler")
upload_dir = os.getenv('MOZ_UPLOAD_DIR')
if not upload_dir:
self.log.critical("Profiling ignored because MOZ_UPLOAD_DIR was not set")
else:
self.gecko_profiler = GeckoProfile(upload_dir,
self.config,
test)
def wait_for_test_finish(self, test, timeout):
# convert timeout to seconds and account for page cycles
timeout = int(timeout / 1000) * int(test['page_cycles'])
# account for the pause the raptor webext runner takes after browser startup
timeout += (int(self.post_startup_delay / 1000) + 3)
# if geckoProfile enabled, give browser more time for profiling
if self.config['gecko_profile'] is True:
timeout += 5 * 60
elapsed_time = 0
while not self.control_server._finished:
time.sleep(1)
# we only want to force browser-shutdown on timeout if not in debug mode;
# in debug-mode we leave the browser running (require manual shutdown)
if not self.debug_mode:
elapsed_time += 1
if elapsed_time > (timeout) - 5: # stop 5 seconds early
self.log.info("application timed out after {} seconds".format(timeout))
self.control_server.wait_for_quit()
break
def process_results(self, test_names):
# when running locally output results in build/raptor.json; when running
# in production output to a local.json to be turned into tc job artifact
if self.config.get('run_local', False):
if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
raptor_json_path = os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
'testing', 'mozharness', 'build', 'raptor.json')
else:
raptor_json_path = os.path.join(here, 'raptor.json')
else:
raptor_json_path = os.path.join(os.getcwd(), 'local.json')
self.config['raptor_json_path'] = raptor_json_path
return self.results_handler.summarize_and_output(self.config, test_names)
def get_page_timeout_list(self):
return self.results_handler.page_timeout_list
def clean_up(self):
self.control_server.stop()
if self.config['app'] not in self.firefox_android_apps:
self.runner.stop()
elif self.config['app'] in self.firefox_android_apps:
self.log.info('removing reverse socket connections')
self.device.remove_socket_connections('reverse')
else:
pass
self.log.info("finished")
def start_control_server(self):
self.control_server = RaptorControlServer(self.results_handler)
self.control_server.start()
class Raptor(object):
"""Container class for Raptor"""
def __init__(self, app, binary, run_local=False, obj_path=None):
self.config = {}
self.config['app'] = app
self.config['binary'] = binary
self.config['platform'] = mozinfo.os
self.config['processor'] = mozinfo.processor
self.config['run_local'] = run_local
self.config['obj_path'] = obj_path
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
self.log = get_default_logger(component='raptor-main')
self.control_server = None
self.playback = None
self.benchmark = None
# Create the profile
self.profile = create_profile(self.config['app'])
# Merge in base profiles
with open(os.path.join(self.profile_data_dir, 'profiles.json'),
'r') as fh:
base_profiles = json.load(fh)['raptor']
for name in base_profiles:
path = os.path.join(self.profile_data_dir, name)
self.log.info("Merging profile: {}".format(path))
self.profile.merge(path)
# create results holder
self.results_handler = RaptorResultsHandler()
# Create the runner
self.output_handler = OutputHandler()
process_args = {
'processOutputLine': [self.output_handler],
}
runner_cls = runners[app]
self.runner = runner_cls(binary,
profile=self.profile,
process_args=process_args)
self.log.info("raptor config: %s" % str(self.config))
@property
def profile_data_dir(self):
if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
'testing', 'profiles')
if build:
return os.path.join(build.topsrcdir, 'testing', 'profiles')
return os.path.join(here, 'profile_data')
def start_control_server(self):
self.control_server = RaptorControlServer(self.results_handler)
self.control_server.start()
def get_playback_config(self, test):
self.config['playback_tool'] = test.get('playback')
self.log.info("test uses playback tool: %s " %
self.config['playback_tool'])
self.config['playback_binary_manifest'] = test.get(
'playback_binary_manifest', None)
_key = 'playback_binary_zip_%s' % self.config['platform']
self.config['playback_binary_zip'] = test.get(_key, None)
self.config['playback_pageset_manifest'] = test.get(
'playback_pageset_manifest', None)
_key = 'playback_pageset_zip_%s' % self.config['platform']
self.config['playback_pageset_zip'] = test.get(_key, None)
self.config['playback_recordings'] = test.get('playback_recordings',
None)
self.config['python3_win_manifest'] = test.get('python3_win_manifest',
None)
def run_test(self, test, timeout=None):
self.log.info("starting raptor test: %s" % test['name'])
self.log.info("test settings: %s" % str(test))
self.log.info("raptor config: %s" % str(self.config))
# benchmark-type tests require the benchmark test to be served out
if test.get('type') == "benchmark":
self.benchmark = Benchmark(self.config, test)
benchmark_port = int(self.benchmark.port)
else:
benchmark_port = 0
gen_test_config(self.config['app'], test['name'],
self.control_server.port, benchmark_port)
# must intall raptor addon each time because we dynamically update some content
raptor_webext = os.path.join(webext_dir, 'raptor')
self.log.info("installing webext %s" % raptor_webext)
self.profile.addons.install(raptor_webext)
# on firefox we can get an addon id; chrome addon actually is just cmd line arg
if self.config['app'] == "firefox":
webext_id = self.profile.addons.addon_details(raptor_webext)['id']
# some tests require tools to playback the test pages
if test.get('playback', None) is not None:
self.get_playback_config(test)
# startup the playback tool
self.playback = get_playback(self.config)
self.runner.start()
proc = self.runner.process_handler
self.output_handler.proc = proc
self.control_server.browser_proc = proc
self.control_server._finished = False
# convert to seconds and account for page cycles
timeout = int(timeout / 1000) * int(test['page_cycles'])
try:
elapsed_time = 0
while not self.control_server._finished:
time.sleep(1)
elapsed_time += 1
if elapsed_time > (timeout) - 5: # stop 5 seconds early
self.control_server.wait_for_quit()
break
finally:
try:
self.runner.check_for_crashes()
except NotImplementedError: # not implemented for Chrome
pass
if self.playback is not None:
self.playback.stop()
# remove the raptor webext; as it must be reloaded with each subtest anyway
# applies to firefox only; chrome the addon is actually just cmd line arg
if self.config['app'] == "firefox":
self.log.info("removing webext %s" % raptor_webext)
self.profile.addons.remove_addon(webext_id)
if self.runner.is_running():
self.log.info(
"Application timed out after {} seconds".format(timeout))
self.runner.stop()
def process_results(self):
# when running locally output results in build/raptor.json; when running
# in production output to a local.json to be turned into tc job artifact
if self.config.get('run_local', False):
if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
raptor_json_path = os.path.join(
os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing',
'mozharness', 'build', 'raptor.json')
else:
raptor_json_path = os.path.join(here, 'raptor.json')
else:
raptor_json_path = os.path.join(os.getcwd(), 'local.json')
self.config['raptor_json_path'] = raptor_json_path
return self.results_handler.summarize_and_output(self.config)
def clean_up(self):
self.control_server.stop()
self.runner.stop()
self.log.info("finished")
class Raptor(object):
"""Container class for Raptor"""
def __init__(self,
app,
binary,
run_local=False,
obj_path=None,
gecko_profile=False,
gecko_profile_interval=None,
gecko_profile_entries=None,
symbols_path=None,
host=None,
is_release_build=False,
debug_mode=False):
self.config = {}
self.config['app'] = app
self.config['binary'] = binary
self.config['platform'] = mozinfo.os
self.config['processor'] = mozinfo.processor
self.config['run_local'] = run_local
self.config['obj_path'] = obj_path
self.config['gecko_profile'] = gecko_profile
self.config['gecko_profile_interval'] = gecko_profile_interval
self.config['gecko_profile_entries'] = gecko_profile_entries
self.config['symbols_path'] = symbols_path
self.config['host'] = host
self.config['is_release_build'] = is_release_build
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
self.log = get_default_logger(component='raptor-main')
self.control_server = None
self.playback = None
self.benchmark = None
self.gecko_profiler = None
self.post_startup_delay = 30000
self.device = None
# debug mode is currently only supported when running locally
self.debug_mode = debug_mode if self.config['run_local'] else False
# if running debug-mode reduce the pause after browser startup
if self.debug_mode:
self.post_startup_delay = 3000
self.log.info(
"debug-mode enabled, reducing post-browser startup pause to %d ms"
% self.post_startup_delay)
# Create the profile; for geckoview we want a firefox profile type
if self.config['app'] == 'geckoview':
self.profile = create_profile('firefox')
else:
self.profile = create_profile(self.config['app'])
# Merge in base profiles
with open(os.path.join(self.profile_data_dir, 'profiles.json'),
'r') as fh:
base_profiles = json.load(fh)['raptor']
for name in base_profiles:
path = os.path.join(self.profile_data_dir, name)
self.log.info("Merging profile: {}".format(path))
self.profile.merge(path)
# add profile dir to our config
self.config['local_profile_dir'] = self.profile.profile
# create results holder
self.results_handler = RaptorResultsHandler()
# when testing desktop browsers we use mozrunner to start the browser; when
# testing on android (i.e. geckoview) we use mozdevice to control the device app
if self.config['app'] == "geckoview":
# create the android device handler; it gets initiated and sets up adb etc
self.log.info("creating android device handler using mozdevice")
self.device = ADBAndroid(verbose=True)
self.device.clear_logcat()
else:
# create the desktop browser runner
self.log.info("creating browser runner using mozrunner")
self.output_handler = OutputHandler()
process_args = {
'processOutputLine': [self.output_handler],
}
runner_cls = runners[app]
self.runner = runner_cls(binary,
profile=self.profile,
process_args=process_args)
self.log.info("raptor config: %s" % str(self.config))
@property
def profile_data_dir(self):
if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
'testing', 'profiles')
if build:
return os.path.join(build.topsrcdir, 'testing', 'profiles')
return os.path.join(here, 'profile_data')
def start_control_server(self):
self.control_server = RaptorControlServer(self.results_handler,
self.debug_mode)
self.control_server.start()
# for android we must make the control server available to the device
if self.config['app'] == "geckoview" and self.config['host'] in (
'localhost', '127.0.0.1'):
self.log.info(
"making the raptor control server port available to device")
_tcp_port = "tcp:%s" % self.control_server.port
self.device.create_socket_connection('reverse', _tcp_port,
_tcp_port)
def get_playback_config(self, test):
self.config['playback_tool'] = test.get('playback')
self.log.info("test uses playback tool: %s " %
self.config['playback_tool'])
self.config['playback_binary_manifest'] = test.get(
'playback_binary_manifest', None)
_key = 'playback_binary_zip_%s' % self.config['platform']
self.config['playback_binary_zip'] = test.get(_key, None)
self.config['playback_pageset_manifest'] = test.get(
'playback_pageset_manifest', None)
_key = 'playback_pageset_zip_%s' % self.config['platform']
self.config['playback_pageset_zip'] = test.get(_key, None)
self.config['playback_recordings'] = test.get('playback_recordings',
None)
self.config['python3_win_manifest'] = test.get('python3_win_manifest',
None)
def run_test(self, test, timeout=None):
self.log.info("starting raptor test: %s" % test['name'])
self.log.info("test settings: %s" % str(test))
self.log.info("raptor config: %s" % str(self.config))
# benchmark-type tests require the benchmark test to be served out
if test.get('type') == "benchmark":
self.benchmark = Benchmark(self.config, test)
benchmark_port = int(self.benchmark.port)
# for android we must make the benchmarks server available to the device
if self.config['app'] == "geckoview" and self.config['host'] \
in ('localhost', '127.0.0.1'):
self.log.info(
"making the raptor benchmarks server port available to device"
)
_tcp_port = "tcp:%s" % benchmark_port
self.device.create_socket_connection('reverse', _tcp_port,
_tcp_port)
else:
benchmark_port = 0
gen_test_config(self.config['app'],
test['name'],
self.control_server.port,
self.post_startup_delay,
host=self.config['host'],
b_port=benchmark_port,
debug_mode=1 if self.debug_mode else 0)
# must intall raptor addon each time because we dynamically update some content
# note: for chrome the addon is just a list of paths that ultimately are added
# to the chromium command line '--load-extension' argument
raptor_webext = os.path.join(webext_dir, 'raptor')
self.log.info("installing webext %s" % raptor_webext)
self.profile.addons.install(raptor_webext)
# add test specific preferences
if test.get("preferences", None) is not None:
if self.config['app'] == "firefox":
self.profile.set_preferences(json.loads(test['preferences']))
else:
self.log.info("preferences were configured for the test, \
but we do not install them on non Firefox browsers."
)
# on firefox we can get an addon id; chrome addon actually is just cmd line arg
if self.config['app'] in ["firefox", "geckoview"]:
webext_id = self.profile.addons.addon_details(raptor_webext)['id']
# for android/geckoview, create a top-level raptor folder on the device
# sdcard; if it already exists remove it so we start fresh each time
if self.config['app'] == "geckoview":
self.device_raptor_dir = "/sdcard/raptor"
self.config['device_raptor_dir'] = self.device_raptor_dir
if self.device.is_dir(self.device_raptor_dir):
self.log.info("deleting existing device raptor dir: %s" %
self.device_raptor_dir)
self.device.rm(self.device_raptor_dir, recursive=True)
self.log.info("creating raptor folder on sdcard: %s" %
self.device_raptor_dir)
self.device.mkdir(self.device_raptor_dir)
self.device.chmod(self.device_raptor_dir, recursive=True)
# some tests require tools to playback the test pages
if test.get('playback', None) is not None:
self.get_playback_config(test)
# startup the playback tool
self.playback = get_playback(self.config, self.device)
# for android we must make the playback server available to the device
if self.config['app'] == "geckoview" and self.config['host'] \
in ('localhost', '127.0.0.1'):
self.log.info(
"making the raptor playback server port available to device"
)
_tcp_port = "tcp:8080"
self.device.create_socket_connection('reverse', _tcp_port,
_tcp_port)
if self.config['app'] in ("geckoview", "firefox") and \
self.config['host'] not in ('localhost', '127.0.0.1'):
# Must delete the proxy settings from the profile if running
# the test with a host different from localhost.
userjspath = os.path.join(self.profile.profile, 'user.js')
with open(userjspath) as userjsfile:
prefs = userjsfile.readlines()
prefs = [pref for pref in prefs if 'network.proxy' not in pref]
with open(userjspath, 'w') as userjsfile:
userjsfile.writelines(prefs)
# for geckoview/android pageload playback we can't use a policy to turn on the
# proxy; we need to set prefs instead; note that the 'host' may be different
# than '127.0.0.1' so we must set the prefs accordingly
if self.config['app'] == "geckoview" and test.get('playback',
None) is not None:
self.log.info(
"setting profile prefs to turn on the geckoview browser proxy")
no_proxies_on = "localhost, 127.0.0.1, %s" % self.config['host']
proxy_prefs = {}
proxy_prefs["network.proxy.type"] = 1
proxy_prefs["network.proxy.http"] = self.config['host']
proxy_prefs["network.proxy.http_port"] = 8080
proxy_prefs["network.proxy.ssl"] = self.config['host']
proxy_prefs["network.proxy.ssl_port"] = 8080
proxy_prefs["network.proxy.no_proxies_on"] = no_proxies_on
self.profile.set_preferences(proxy_prefs)
# now some final settings, and then startup of the browser under test
if self.config['app'] == "geckoview":
# for android/geckoview we must copy the profile onto the device and set perms
if not self.device.is_app_installed(self.config['binary']):
raise Exception('%s is not installed' % self.config['binary'])
self.device_profile = os.path.join(self.device_raptor_dir,
"profile")
if self.device.is_dir(self.device_profile):
self.log.info("deleting existing device profile folder: %s" %
self.device_profile)
self.device.rm(self.device_profile, recursive=True)
self.log.info("creating profile folder on device: %s" %
self.device_profile)
self.device.mkdir(self.device_profile)
self.log.info("copying firefox profile onto the device")
self.log.info("note: the profile folder being copied is: %s" %
self.profile.profile)
self.log.info(
'the adb push cmd copies that profile dir to a new temp dir before copy'
)
self.device.push(self.profile.profile, self.device_profile)
self.device.chmod(self.device_profile, recursive=True)
# now start the geckoview app
self.log.info("starting %s" % self.config['app'])
extra_args = [
"-profile", self.device_profile, "--es", "env0",
"LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6"
]
try:
# make sure the geckoview app is not running before
# attempting to start.
self.device.stop_application(self.config['binary'])
self.device.launch_activity(self.config['binary'],
"GeckoViewActivity",
extra_args=extra_args,
url='about:blank',
e10s=True,
fail_if_running=False)
except Exception:
self.log.error("Exception launching %s" %
self.config['binary'])
raise
self.control_server.device = self.device
self.control_server.app_name = self.config['binary']
else:
# For Firefox we need to set
# MOZ_DISABLE_NONLOCAL_CONNECTIONS=1 env var before
# startup when testing release builds from mozilla-beta or
# mozilla-release. This is because of restrictions on
# release builds that require webextensions to be signed
# unless MOZ_DISABLE_NONLOCAL_CONNECTIONS is set to '1'.
if self.config['app'] == "firefox" and self.config[
'is_release_build']:
self.log.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=1")
os.environ['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = "1"
# if running debug-mode, tell Firefox to open the browser console on startup
# for google chrome, open the devtools on the raptor test tab
if self.debug_mode:
if self.config['app'] == "firefox":
self.runner.cmdargs.extend(['-jsconsole'])
if self.config['app'] == "chrome":
self.runner.cmdargs.extend(
['--auto-open-devtools-for-tabs'])
# now start the desktop browser
self.log.info("starting %s" % self.config['app'])
# if running a pageload test on google chrome, add the cmd line options
# to turn on the proxy and ignore security certificate errors
# if using host localhost, 127.0.0.1.
if self.config['app'] == "chrome" and test.get('playback',
None) is not None:
chrome_args = [
'--proxy-server="http=127.0.0.1:8080;' +
'https=127.0.0.1:8080;ssl=127.0.0.1:8080"',
'--ignore-certificate-errors'
]
if self.config['host'] not in ('localhost', '127.0.0.1'):
chrome_args[0] = chrome_args[0].replace(
'127.0.0.1', self.config['host'])
if ' '.join(chrome_args) not in ' '.join(self.runner.cmdargs):
self.runner.cmdargs.extend(chrome_args)
self.runner.start()
proc = self.runner.process_handler
self.output_handler.proc = proc
self.control_server.browser_proc = proc
# pageload tests need to be able to access non-local connections via mitmproxy
if self.config['app'] == "firefox" and self.config['is_release_build'] and \
test.get('playback', None) is not None:
self.log.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=0")
os.environ['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = "0"
# if geckoProfile is enabled, initialize it
if self.config['gecko_profile'] is True:
self._init_gecko_profiling(test)
# tell the control server the gecko_profile dir; the control server will
# receive the actual gecko profiles from the web ext and will write them
# to disk; then profiles are picked up by gecko_profile.symbolicate
self.control_server.gecko_profile_dir = self.gecko_profiler.gecko_profile_dir
# set our cs flag to indicate we are running the browser/app
self.control_server._finished = False
# convert to seconds and account for page cycles
timeout = int(timeout / 1000) * int(test['page_cycles'])
# account for the pause the raptor webext runner takes after browser startup
timeout += int(self.post_startup_delay / 1000)
# if geckoProfile enabled, give browser more time for profiling
if self.config['gecko_profile'] is True:
timeout += 5 * 60
try:
elapsed_time = 0
while not self.control_server._finished:
time.sleep(1)
# we only want to force browser-shutdown on timeout if not in debug mode;
# in debug-mode we leave the browser running (require manual shutdown)
if not self.debug_mode:
elapsed_time += 1
if elapsed_time > (timeout) - 5: # stop 5 seconds early
self.log.info(
"application timed out after {} seconds".format(
timeout))
self.control_server.wait_for_quit()
break
finally:
if self.config['app'] != "geckoview":
try:
self.runner.check_for_crashes()
except NotImplementedError: # not implemented for Chrome
pass
# TODO: if on geckoview is there some cleanup here i.e. check for crashes?
if self.playback is not None:
self.playback.stop()
# remove the raptor webext; as it must be reloaded with each subtest anyway
self.log.info("removing webext %s" % raptor_webext)
if self.config['app'] in ["firefox", "geckoview"]:
self.profile.addons.remove_addon(webext_id)
# for chrome the addon is just a list (appended to cmd line)
if self.config['app'] in ["chrome", "chrome-android"]:
self.profile.addons.remove(raptor_webext)
# gecko profiling symbolication
if self.config['gecko_profile'] is True:
self.gecko_profiler.symbolicate()
# clean up the temp gecko profiling folders
self.log.info("cleaning up after gecko profiling")
self.gecko_profiler.clean()
# browser should be closed by now but this is a backup-shutdown (if not in debug-mode)
if not self.debug_mode:
if self.config['app'] != "geckoview":
if self.runner.is_running():
self.runner.stop()
# TODO the geckoview app should have been shutdown by this point by the
# control server, but we can double-check here to make sure
else:
# in debug mode, and running locally, leave the browser running
if self.config['run_local']:
self.log.info(
"* debug-mode enabled - please shutdown the browser manually..."
)
self.runner.wait(timeout=None)
def _init_gecko_profiling(self, test):
self.log.info("initializing gecko profiler")
upload_dir = os.getenv('MOZ_UPLOAD_DIR')
if not upload_dir:
self.log.critical(
"Profiling ignored because MOZ_UPLOAD_DIR was not set")
else:
self.gecko_profiler = GeckoProfile(upload_dir, self.config, test)
def process_results(self):
# when running locally output results in build/raptor.json; when running
# in production output to a local.json to be turned into tc job artifact
if self.config.get('run_local', False):
if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
raptor_json_path = os.path.join(
os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing',
'mozharness', 'build', 'raptor.json')
else:
raptor_json_path = os.path.join(here, 'raptor.json')
else:
raptor_json_path = os.path.join(os.getcwd(), 'local.json')
self.config['raptor_json_path'] = raptor_json_path
return self.results_handler.summarize_and_output(self.config)
def get_page_timeout_list(self):
return self.results_handler.page_timeout_list
def clean_up(self):
self.control_server.stop()
if self.config['app'] != "geckoview":
self.runner.stop()
elif self.config['app'] == 'geckoview':
self.log.info('removing reverse socket connections')
self.device.remove_socket_connections('reverse')
else:
pass
self.log.info("finished")
class Raptor(object):
"""Container class for Raptor"""
def __init__(self, app, binary):
self.config = {}
self.config['app'] = app
self.config['binary'] = binary
self.config['platform'] = mozinfo.os
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
self.log = get_default_logger(component='raptor')
self.addons_installed = False
self.control_server = None
self.playback = None
# Create the profile
self.profile = create_profile(self.config['app'])
# Merge in base profiles
with open(os.path.join(self.profile_data_dir, 'profiles.json'),
'r') as fh:
base_profiles = json.load(fh)['raptor']
for name in base_profiles:
path = os.path.join(self.profile_data_dir, name)
self.log.info("Merging profile: {}".format(path))
self.profile.merge(path)
# create results holder
self.results_handler = RaptorResultsHandler()
# Create the runner
self.output_handler = OutputHandler()
process_args = {
'processOutputLine': [self.output_handler],
}
runner_cls = runners[app]
self.runner = runner_cls(binary,
profile=self.profile,
process_args=process_args)
@property
def profile_data_dir(self):
if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
'testing', 'profiles')
if build:
return os.path.join(build.topsrcdir, 'testing', 'profiles')
return os.path.join(here, 'profile_data')
def start_control_server(self):
self.control_server = RaptorControlServer(self.results_handler)
self.control_server.start()
def get_playback_config(self, test):
self.config['playback_tool'] = test.get('playback')
self.log.info("test uses playback tool: %s " %
self.config['playback_tool'])
self.config['playback_binary_manifest'] = test.get(
'playback_binary_manifest', None)
_key = 'playback_binary_zip_%s' % self.config['platform']
self.config['playback_binary_zip'] = test.get(_key, None)
self.config['playback_pageset_manifest'] = test.get(
'playback_pageset_manifest', None)
_key = 'playback_pageset_zip_%s' % self.config['platform']
self.config['playback_pageset_zip'] = test.get(_key, None)
self.config['playback_recordings'] = test.get('playback_recordings',
None)
def run_test(self, test, timeout=None):
self.log.info("starting raptor test: %s" % test['name'])
gen_test_config(self.config['app'], test['name'],
self.control_server.port)
# must intall raptor addon each time because we dynamically update some content
raptor_webext = os.path.join(webext_dir, 'raptor')
self.log.info("installing webext %s" % raptor_webext)
self.profile.addons.install(raptor_webext)
webext_id = self.profile.addons.addon_details(raptor_webext)['id']
# some tests require tools to playback the test pages
if test.get('playback', None) is not None:
self.get_playback_config(test)
# startup the playback tool
self.playback = get_playback(self.config)
self.runner.start()
proc = self.runner.process_handler
self.output_handler.proc = proc
self.control_server.browser_proc = proc
try:
self.runner.wait(timeout)
finally:
try:
self.runner.check_for_crashes()
except NotImplementedError: # not implemented for Chrome
pass
if self.playback is not None:
self.playback.stop()
# remove the raptor webext; as it must be reloaded with each subtest anyway
self.log.info("removing webext %s" % raptor_webext)
self.profile.addons.remove_addon(webext_id)
if self.runner.is_running():
self.log("Application timed out after {} seconds".format(timeout))
self.runner.stop()
def process_results(self):
return self.results_handler.summarize_and_output(self.config)
def clean_up(self):
self.control_server.stop()
self.runner.stop()
self.log.info("finished")
class Raptor(object):
"""Container class for Raptor"""
def __init__(self,
app,
binary,
run_local=False,
obj_path=None,
gecko_profile=False,
gecko_profile_interval=None,
gecko_profile_entries=None,
symbols_path=None,
host=None,
power_test=False,
memory_test=False,
is_release_build=False,
debug_mode=False,
post_startup_delay=None,
interrupt_handler=None,
**kwargs):
# Override the magic --host HOST_IP with the value of the environment variable.
if host == 'HOST_IP':
host = os.environ['HOST_IP']
self.config = {
'app': app,
'binary': binary,
'platform': mozinfo.os,
'processor': mozinfo.processor,
'run_local': run_local,
'obj_path': obj_path,
'gecko_profile': gecko_profile,
'gecko_profile_interval': gecko_profile_interval,
'gecko_profile_entries': gecko_profile_entries,
'symbols_path': symbols_path,
'host': host,
'power_test': power_test,
'memory_test': memory_test,
'is_release_build': is_release_build,
'enable_control_server_wait': memory_test,
}
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
self.log = get_default_logger(component='raptor-main')
self.control_server = None
self.playback = None
self.benchmark = None
self.benchmark_port = 0
self.gecko_profiler = None
self.post_startup_delay = post_startup_delay
self.device = None
self.profile_class = app
self.firefox_android_apps = FIREFOX_ANDROID_APPS
self.interrupt_handler = interrupt_handler
# debug mode is currently only supported when running locally
self.debug_mode = debug_mode if self.config['run_local'] else False
# if running debug-mode reduce the pause after browser startup
if self.debug_mode:
self.post_startup_delay = min(self.post_startup_delay, 3000)
self.log.info(
"debug-mode enabled, reducing post-browser startup pause to %d ms"
% self.post_startup_delay)
self.log.info("main raptor init, config is: %s" % str(self.config))
# create results holder
self.results_handler = RaptorResultsHandler()
@property
def profile_data_dir(self):
if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
'testing', 'profiles')
if build:
return os.path.join(build.topsrcdir, 'testing', 'profiles')
return os.path.join(here, 'profile_data')
def check_for_crashes(self):
raise NotImplementedError
def run_test_setup(self, test):
self.log.info("starting raptor test: %s" % test['name'])
self.log.info("test settings: %s" % str(test))
self.log.info("raptor config: %s" % str(self.config))
if test.get('type') == "benchmark":
self.serve_benchmark_source(test)
gen_test_config(
self.config['app'],
test['name'],
self.control_server.port,
self.post_startup_delay,
host=self.config['host'],
b_port=self.benchmark_port,
debug_mode=1 if self.debug_mode else 0,
browser_cycle=test.get('browser_cycle', 1),
)
self.install_raptor_webext()
if test.get("preferences") is not None:
self.set_browser_test_prefs(test['preferences'])
# if 'alert_on' was provided in the test INI, add to our config for results/output
self.config['subtest_alert_on'] = test.get('alert_on')
def run_test_teardown(self):
self.check_for_crashes()
if self.playback is not None:
self.playback.stop()
self.remove_raptor_webext()
# gecko profiling symbolication
if self.config['gecko_profile'] is True:
self.gecko_profiler.symbolicate()
# clean up the temp gecko profiling folders
self.log.info("cleaning up after gecko profiling")
self.gecko_profiler.clean()
def set_browser_test_prefs(self, raw_prefs):
# add test specific preferences
self.log.info("preferences were configured for the test, however \
we currently do not install them on non Firefox browsers."
)
def create_browser_profile(self):
self.profile = create_profile(self.profile_class)
# Merge in base profiles
with open(os.path.join(self.profile_data_dir, 'profiles.json'),
'r') as fh:
base_profiles = json.load(fh)['raptor']
for name in base_profiles:
path = os.path.join(self.profile_data_dir, name)
self.log.info("Merging profile: {}".format(path))
self.profile.merge(path)
# add profile dir to our config
self.config['local_profile_dir'] = self.profile.profile
def start_control_server(self):
self.control_server = RaptorControlServer(self.results_handler,
self.debug_mode)
self.control_server.start()
if self.config['enable_control_server_wait']:
self.control_server_wait_set(
'webext_status/__raptor_shutdownBrowser')
def get_playback_config(self, test):
platform = self.config['platform']
playback_dir = os.path.join(here, 'playback')
self.config.update({
'playback_tool':
test.get('playback'),
'playback_version':
test.get('playback_version', "2.0.2"),
'playback_binary_zip':
test.get('playback_binary_zip_%s' % platform),
'playback_pageset_zip':
test.get('playback_pageset_zip_%s' % platform),
'playback_binary_manifest':
test.get('playback_binary_manifest'),
'playback_pageset_manifest':
test.get('playback_pageset_manifest'),
})
for key in ('playback_pageset_manifest', 'playback_pageset_zip'):
if self.config.get(key) is None:
continue
self.config[key] = os.path.join(playback_dir, self.config[key])
self.log.info("test uses playback tool: %s " %
self.config['playback_tool'])
def serve_benchmark_source(self, test):
# benchmark-type tests require the benchmark test to be served out
self.benchmark = Benchmark(self.config, test)
self.benchmark_port = int(self.benchmark.port)
def install_raptor_webext(self):
# must intall raptor addon each time because we dynamically update some content
# the webext is installed into the browser profile
# note: for chrome the addon is just a list of paths that ultimately are added
# to the chromium command line '--load-extension' argument
self.raptor_webext = os.path.join(webext_dir, 'raptor')
self.log.info("installing webext %s" % self.raptor_webext)
self.profile.addons.install(self.raptor_webext)
# on firefox we can get an addon id; chrome addon actually is just cmd line arg
try:
self.webext_id = self.profile.addons.addon_details(
self.raptor_webext)['id']
except AttributeError:
self.webext_id = None
def remove_raptor_webext(self):
# remove the raptor webext; as it must be reloaded with each subtest anyway
self.log.info("removing webext %s" % self.raptor_webext)
if self.config['app'] in [
'firefox', 'geckoview', 'fennec', 'refbrow', 'fenix'
]:
self.profile.addons.remove_addon(self.webext_id)
# for chrome the addon is just a list (appended to cmd line)
chrome_apps = CHROMIUM_DISTROS + ["chrome-android", "chromium-android"]
if self.config['app'] in chrome_apps:
self.profile.addons.remove(self.raptor_webext)
def get_proxy_command_for_mitm(self, test, version):
# Generate Mitmproxy playback args
script = os.path.join(here, "playback",
"alternate-server-replay-{}.py".format(version))
recordings = test.get("playback_recordings")
if recordings:
recording_paths = []
proxy_dir = self.playback.mozproxy_dir
for recording in recordings.split():
if not recording:
continue
recording_paths.append(os.path.join(proxy_dir, recording))
# this part is platform-specific
if mozinfo.os == "win":
script = script.replace("\\", "\\\\\\")
recording_paths = [
recording_path.replace("\\", "\\\\\\")
for recording_path in recording_paths
]
if version == "2.0.2":
self.playback.config['playback_tool_args'] = [
"--replay-kill-extra", "--script",
'""{} {}""'.format(script, " ".join(recording_paths))
]
elif version == "4.0.4":
self.playback.config['playback_tool_args'] = [
"--scripts", script, "--set",
"server_replay={}".format(" ".join(recording_paths))
]
else:
raise Exception("Mitmproxy version is unknown!")
def start_playback(self, test):
# creating the playback tool
self.get_playback_config(test)
self.playback = get_playback(self.config, self.device)
self.get_proxy_command_for_mitm(test, self.config['playback_version'])
# let's start it!
self.playback.start()
def delete_proxy_settings_from_profile(self):
# Must delete the proxy settings from the profile if running
# the test with a host different from localhost.
userjspath = os.path.join(self.profile.profile, 'user.js')
with open(userjspath) as userjsfile:
prefs = userjsfile.readlines()
prefs = [pref for pref in prefs if 'network.proxy' not in pref]
with open(userjspath, 'w') as userjsfile:
userjsfile.writelines(prefs)
def _init_gecko_profiling(self, test):
self.log.info("initializing gecko profiler")
upload_dir = os.getenv('MOZ_UPLOAD_DIR')
if not upload_dir:
self.log.critical(
"Profiling ignored because MOZ_UPLOAD_DIR was not set")
else:
self.gecko_profiler = GeckoProfile(upload_dir, self.config, test)
def wait_for_test_finish(self, test, timeout):
# convert timeout to seconds and account for page cycles
timeout = int(timeout / 1000) * int(test.get('page_cycles', 1))
# account for the pause the raptor webext runner takes after browser startup
timeout += (int(self.post_startup_delay / 1000) + 3)
# if geckoProfile enabled, give browser more time for profiling
if self.config['gecko_profile'] is True:
timeout += 5 * 60
elapsed_time = 0
while not self.control_server._finished:
if self.config['enable_control_server_wait']:
response = self.control_server_wait_get()
if response == 'webext_status/__raptor_shutdownBrowser':
if self.config['memory_test']:
generate_android_memory_profile(self, test['name'])
self.control_server_wait_continue()
time.sleep(1)
# we only want to force browser-shutdown on timeout if not in debug mode;
# in debug-mode we leave the browser running (require manual shutdown)
if not self.debug_mode:
elapsed_time += 1
if elapsed_time > (timeout) - 5: # stop 5 seconds early
self.log.info(
"application timed out after {} seconds".format(
timeout))
self.control_server.wait_for_quit()
break
def process_results(self, test_names):
# when running locally output results in build/raptor.json; when running
# in production output to a local.json to be turned into tc job artifact
if self.config.get('run_local', False):
if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
raptor_json_path = os.path.join(
os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing',
'mozharness', 'build', 'raptor.json')
else:
raptor_json_path = os.path.join(here, 'raptor.json')
else:
raptor_json_path = os.path.join(os.getcwd(), 'local.json')
self.config['raptor_json_path'] = raptor_json_path
return self.results_handler.summarize_and_output(
self.config, test_names)
def get_page_timeout_list(self):
return self.results_handler.page_timeout_list
def clean_up(self):
if self.config['enable_control_server_wait']:
self.control_server_wait_clear('all')
self.control_server.stop()
self.log.info("finished")
def control_server_wait_set(self, state):
response = requests.post("http://127.0.0.1:%s/" %
self.control_server.port,
json={
"type": "wait-set",
"data": state
})
return response.content
def control_server_wait_timeout(self, timeout):
response = requests.post("http://127.0.0.1:%s/" %
self.control_server.port,
json={
"type": "wait-timeout",
"data": timeout
})
return response.content
def control_server_wait_get(self):
response = requests.post("http://127.0.0.1:%s/" %
self.control_server.port,
json={
"type": "wait-get",
"data": ""
})
return response.content
def control_server_wait_continue(self):
response = requests.post("http://127.0.0.1:%s/" %
self.control_server.port,
json={
"type": "wait-continue",
"data": ""
})
return response.content
def control_server_wait_clear(self, state):
response = requests.post("http://127.0.0.1:%s/" %
self.control_server.port,
json={
"type": "wait-clear",
"data": state
})
return response.content
def start_control_server(self):
self.control_server = RaptorControlServer()
self.control_server.start()
class Raptor(object):
"""Container class for Raptor"""
def __init__(self, app, binary):
self.config = {}
self.config['app'] = app
self.config['binary'] = binary
self.config['platform'] = mozinfo.os
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
self.log = get_default_logger(component='raptor')
self.control_server = None
self.playback = None
# Create the profile
pref_file = os.path.join(here, 'preferences',
'{}.json'.format(self.config['app']))
prefs = {}
if os.path.isfile(pref_file):
with open(pref_file, 'r') as fh:
prefs = json.load(fh)
try:
self.profile = create_profile(self.config['app'],
preferences=prefs)
except NotImplementedError:
self.profile = None
# Create the runner
self.output_handler = OutputHandler()
process_args = {
'processOutputLine': [self.output_handler],
}
runner_cls = runners[app]
self.runner = runner_cls(binary,
profile=self.profile,
process_args=process_args)
def start_control_server(self):
self.control_server = RaptorControlServer()
self.control_server.start()
def get_playback_config(self, test):
self.config['playback_tool'] = test.get('playback')
self.log.info("test uses playback tool: %s " %
self.config['playback_tool'])
self.config['playback_binary_manifest'] = test.get(
'playback_binary_manifest', None)
_key = 'playback_binary_zip_%s' % self.config['platform']
self.config['playback_binary_zip'] = test.get(_key, None)
self.config['playback_pageset_manifest'] = test.get(
'playback_pageset_manifest', None)
_key = 'playback_pageset_zip_%s' % self.config['platform']
self.config['playback_pageset_zip'] = test.get(_key, None)
self.config['playback_recordings'] = test.get('playback_recordings',
None)
def run_test(self, test, timeout=None):
self.log.info("starting raptor test: %s" % test['name'])
gen_test_config(self.config['app'], test['name'],
self.control_server.port)
self.profile.addons.install(os.path.join(webext_dir, 'raptor'))
# some tests require tools to playback the test pages
if test.get('playback', None) is not None:
self.get_playback_config(test)
# startup the playback tool
self.playback = get_playback(self.config)
self.runner.start()
first_time = int(time.time()) * 1000
proc = self.runner.process_handler
self.output_handler.proc = proc
try:
self.runner.wait(timeout)
finally:
try:
self.runner.check_for_crashes()
except NotImplementedError: # not implemented for Chrome
pass
if self.playback is not None:
self.playback.stop()
if self.runner.is_running():
self.log("Application timed out after {} seconds".format(timeout))
self.runner.stop()
proc.output.append(
"__startBeforeLaunchTimestamp%d__endBeforeLaunchTimestamp" %
first_time)
proc.output.append(
"__startAfterTerminationTimestamp%d__endAfterTerminationTimestamp"
% (int(time.time()) * 1000))
def process_results(self):
self.log.info('todo: process results and dump in PERFHERDER_JSON blob')
self.log.info('- or - do we want the control server to do that?')
def clean_up(self):
self.control_server.stop()
self.runner.stop()
self.log.info("raptor finished")