def run_test(self, test, timeout=None):
if self.config['power_test']:
init_geckoview_power_test(self)
self.run_test_setup(test)
self.create_raptor_sdcard_folder()
if test.get('playback', None) is not None:
self.start_playback(test)
if self.config['host'] not in ('localhost', '127.0.0.1'):
self.delete_proxy_settings_from_profile()
if test.get('playback', None) is not None:
self.turn_on_android_app_proxy()
self.copy_profile_onto_device()
# now start the browser/app under test
self.launch_firefox_android_app()
# set our control server flag to indicate we are running the browser/app
self.control_server._finished = False
self.wait_for_test_finish(test, timeout)
if self.config['power_test']:
finish_geckoview_power_test(self)
self.run_test_teardown()
# in debug mode, and running locally, leave the browser running
if self.debug_mode and self.config['run_local']:
self.log.info("* debug-mode enabled - please shutdown the browser manually...")
self.runner.wait(timeout=None)
def launch_firefox_android_app(self):
self.log.info("starting %s" % self.config['app'])
extra_args = ["-profile", self.device_profile,
"--es", "env0", "LOG_VERBOSE=1",
"--es", "env1", "R_LOG_LEVEL=6"]
try:
# make sure the android app is not already running
self.device.stop_application(self.config['binary'])
if self.config['app'] == "fennec":
self.device.launch_fennec(self.config['binary'],
extra_args=extra_args,
url='about:blank',
fail_if_running=False)
else:
self.device.launch_activity(self.config['binary'],
self.config['activity'],
extra_args=extra_args,
url='about:blank',
e10s=True,
fail_if_running=False)
except Exception as e:
self.log.error("Exception launching %s" % self.config['binary'])
self.log.error("Exception: %s %s" % (type(e).__name__, str(e)))
if self.config['power_test']:
finish_geckoview_power_test(self)
raise
# give our control server the device and app info
self.control_server.device = self.device
self.control_server.app_name = self.config['binary']
def run_test_cold(self, test, timeout=None):
'''
Run the Raptor test but restart the entire browser app between page-cycles.
Note: For page-load tests, playback will only be started once - at the beginning of all
browser cycles, and then stopped after all cycles are finished. The proxy is set via prefs
in the browser profile so those will need to be set again in each new profile/cycle.
Note that instead of using the certutil tool each time to create a db and import the
mitmproxy SSL cert (it's done in mozbase/mozproxy) we will simply copy the existing
cert db from the first cycle's browser profile into the new clean profile; this way
we don't have to re-create the cert db on each browser cycle.
Since we're running in cold-mode, before this point (in manifest.py) the
'expected-browser-cycles' value was already set to the initial 'page-cycles' value;
and the 'page-cycles' value was set to 1 as we want to perform one page-cycle per
browser restart.
The 'browser-cycle' value is the current overall browser start iteration. The control
server will receive the current 'browser-cycle' and the 'expected-browser-cycles' in
each results set received; and will pass that on as part of the results so that the
results processing will know results for multiple browser cycles are being received.
The default will be to run in warm mode; unless 'cold = true' is set in the test INI.
'''
self.log.info(
"test %s is running in cold mode; browser WILL be restarted between "
"page cycles" % test['name'])
if self.config['power_test']:
init_geckoview_power_test(self)
for test['browser_cycle'] in range(1, test['expected_browser_cycles'] +
1):
self.log.info("begin browser cycle %d of %d for test %s" %
(test['browser_cycle'],
test['expected_browser_cycles'], test['name']))
self.run_test_setup(test)
if test['browser_cycle'] == 1:
self.create_raptor_sdcard_folder()
if test.get('playback', None) is not None:
self.start_playback(test)
# an ssl cert db has now been created in the profile; copy it out so we
# can use the same cert db in future test cycles / browser restarts
local_cert_db_dir = tempfile.mkdtemp()
self.log.info(
"backing up browser ssl cert db that was created via certutil"
)
self.copy_cert_db(self.config['local_profile_dir'],
local_cert_db_dir)
if self.config['host'] not in ('localhost', '127.0.0.1'):
self.delete_proxy_settings_from_profile()
else:
# double-check to ensure app has been shutdown
self.device.stop_application(self.config['binary'])
# clear the android app data before the next app startup
self.clear_app_data()
# initial browser profile was already created before run_test was called;
# now additional browser cycles we want to create a new one each time
self.create_browser_profile()
# get cert db from previous cycle profile and copy into new clean profile
# this saves us from having to start playback again / recreate cert db etc.
self.log.info(
"copying existing ssl cert db into new browser profile")
self.copy_cert_db(local_cert_db_dir,
self.config['local_profile_dir'])
self.run_test_setup(test)
if test.get('playback', None) is not None:
self.turn_on_android_app_proxy()
self.copy_profile_onto_device()
# now start the browser/app under test
self.launch_firefox_android_app()
# set our control server flag to indicate we are running the browser/app
self.control_server._finished = False
self.wait_for_test_finish(test, timeout)
# in debug mode, and running locally, leave the browser running
if self.debug_mode and self.config['run_local']:
self.log.info(
"* debug-mode enabled - please shutdown the browser manually..."
)
self.runner.wait(timeout=None)
if self.config['power_test']:
finish_geckoview_power_test(self)
self.run_test_teardown()
def run_test(self, test, timeout=None):
self.log.info("starting raptor test: %s" % test['name'])
self.log.info("test settings: %s" % str(test))
self.log.info("raptor config: %s" % str(self.config))
# benchmark-type tests require the benchmark test to be served out
if test.get('type') == "benchmark":
self.benchmark = Benchmark(self.config, test)
benchmark_port = int(self.benchmark.port)
# for android we must make the benchmarks server available to the device
if self.config['app'] in ['geckoview', 'fennec'] and \
self.config['host'] in ('localhost', '127.0.0.1'):
self.log.info(
"making the raptor benchmarks server port available to device"
)
_tcp_port = "tcp:%s" % benchmark_port
self.device.create_socket_connection('reverse', _tcp_port,
_tcp_port)
else:
benchmark_port = 0
gen_test_config(self.config['app'],
test['name'],
self.control_server.port,
self.post_startup_delay,
host=self.config['host'],
b_port=benchmark_port,
debug_mode=1 if self.debug_mode else 0)
# must intall raptor addon each time because we dynamically update some content
# note: for chrome the addon is just a list of paths that ultimately are added
# to the chromium command line '--load-extension' argument
raptor_webext = os.path.join(webext_dir, 'raptor')
self.log.info("installing webext %s" % raptor_webext)
self.profile.addons.install(raptor_webext)
# add test specific preferences
if test.get("preferences", None) is not None:
if self.config['app'] == "firefox":
self.profile.set_preferences(json.loads(test['preferences']))
else:
self.log.info("preferences were configured for the test, \
but we do not install them on non Firefox browsers."
)
# if 'alert_on' was provided in the test INI, we must add that to our config
# for use in our results.py and output.py
# test['alert_on'] has already been converted to a list and stripped of spaces
self.config['subtest_alert_on'] = test.get('alert_on', None)
# on firefox we can get an addon id; chrome addon actually is just cmd line arg
if self.config['app'] in ['firefox', 'geckoview', 'fennec']:
webext_id = self.profile.addons.addon_details(raptor_webext)['id']
# for android/geckoview, create a top-level raptor folder on the device
# sdcard; if it already exists remove it so we start fresh each time
if self.config['app'] in ["geckoview", "fennec"]:
self.device_raptor_dir = "/sdcard/raptor"
self.config['device_raptor_dir'] = self.device_raptor_dir
if self.device.is_dir(self.device_raptor_dir):
self.log.info("deleting existing device raptor dir: %s" %
self.device_raptor_dir)
self.device.rm(self.device_raptor_dir, recursive=True)
self.log.info("creating raptor folder on sdcard: %s" %
self.device_raptor_dir)
self.device.mkdir(self.device_raptor_dir)
self.device.chmod(self.device_raptor_dir, recursive=True)
# some tests require tools to playback the test pages
if test.get('playback', None) is not None:
# startup the playback tool
self.get_playback_config(test)
self.playback = get_playback(self.config, self.device)
# for android we must make the playback server available to the device
if self.config['app'] == "geckoview" and self.config['host'] \
in ('localhost', '127.0.0.1'):
self.log.info(
"making the raptor playback server port available to device"
)
_tcp_port = "tcp:8080"
self.device.create_socket_connection('reverse', _tcp_port,
_tcp_port)
if self.config['app'] in ('geckoview', 'firefox', 'fennec') and \
self.config['host'] not in ('localhost', '127.0.0.1'):
# Must delete the proxy settings from the profile if running
# the test with a host different from localhost.
userjspath = os.path.join(self.profile.profile, 'user.js')
with open(userjspath) as userjsfile:
prefs = userjsfile.readlines()
prefs = [pref for pref in prefs if 'network.proxy' not in pref]
with open(userjspath, 'w') as userjsfile:
userjsfile.writelines(prefs)
# for geckoview/android pageload playback we can't use a policy to turn on the
# proxy; we need to set prefs instead; note that the 'host' may be different
# than '127.0.0.1' so we must set the prefs accordingly
if self.config['app'] == "geckoview" and test.get('playback',
None) is not None:
self.log.info(
"setting profile prefs to turn on the geckoview browser proxy")
no_proxies_on = "localhost, 127.0.0.1, %s" % self.config['host']
proxy_prefs = {}
proxy_prefs["network.proxy.type"] = 1
proxy_prefs["network.proxy.http"] = self.config['host']
proxy_prefs["network.proxy.http_port"] = 8080
proxy_prefs["network.proxy.ssl"] = self.config['host']
proxy_prefs["network.proxy.ssl_port"] = 8080
proxy_prefs["network.proxy.no_proxies_on"] = no_proxies_on
self.profile.set_preferences(proxy_prefs)
# now some final settings, and then startup of the browser under test
if self.config['app'] in ["geckoview", "fennec"]:
# for geckoview/fennec we must copy the profile onto the device and set perms
if not self.device.is_app_installed(self.config['binary']):
raise Exception('%s is not installed' % self.config['binary'])
self.device_profile = os.path.join(self.device_raptor_dir,
"profile")
if self.device.is_dir(self.device_profile):
self.log.info("deleting existing device profile folder: %s" %
self.device_profile)
self.device.rm(self.device_profile, recursive=True)
self.log.info("creating profile folder on device: %s" %
self.device_profile)
self.device.mkdir(self.device_profile)
self.log.info("copying firefox profile onto the device")
self.log.info("note: the profile folder being copied is: %s" %
self.profile.profile)
self.log.info(
'the adb push cmd copies that profile dir to a new temp dir before copy'
)
self.device.push(self.profile.profile, self.device_profile)
self.device.chmod(self.device_profile, recursive=True)
# now start the geckoview/fennec app
self.log.info("starting %s" % self.config['app'])
extra_args = [
"-profile", self.device_profile, "--es", "env0",
"LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6"
]
if self.config['app'] == 'geckoview':
# launch geckoview example app
try:
# make sure the geckoview app is not running before
# attempting to start.
self.device.stop_application(self.config['binary'])
self.device.launch_activity(self.config['binary'],
"GeckoViewActivity",
extra_args=extra_args,
url='about:blank',
e10s=True,
fail_if_running=False)
except Exception:
self.log.error("Exception launching %s" %
self.config['binary'])
if self.config['power_test']:
finish_geckoview_power_test(self)
raise
else:
# launch fennec
try:
# if fennec is already running, shut it down first
self.device.stop_application(self.config['binary'])
self.device.launch_fennec(self.config['binary'],
extra_args=extra_args,
url='about:blank',
fail_if_running=False)
except Exception:
self.log.error("Exception launching %s" %
self.config['binary'])
if self.config['power_test']:
finish_geckoview_power_test(self)
raise
self.control_server.device = self.device
self.control_server.app_name = self.config['binary']
else:
# For Firefox we need to set
# MOZ_DISABLE_NONLOCAL_CONNECTIONS=1 env var before
# startup when testing release builds from mozilla-beta or
# mozilla-release. This is because of restrictions on
# release builds that require webextensions to be signed
# unless MOZ_DISABLE_NONLOCAL_CONNECTIONS is set to '1'.
if self.config['app'] == "firefox" and self.config[
'is_release_build']:
self.log.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=1")
os.environ['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = "1"
# if running debug-mode, tell Firefox to open the browser console on startup
# for google chrome, open the devtools on the raptor test tab
if self.debug_mode:
if self.config['app'] == "firefox":
self.runner.cmdargs.extend(['-jsconsole'])
if self.config['app'] == "chrome":
self.runner.cmdargs.extend(
['--auto-open-devtools-for-tabs'])
# now start the desktop browser
self.log.info("starting %s" % self.config['app'])
# if running a pageload test on google chrome, add the cmd line options
# to turn on the proxy and ignore security certificate errors
# if using host localhost, 127.0.0.1.
if self.config['app'] == "chrome" and test.get('playback',
None) is not None:
chrome_args = [
'--proxy-server=127.0.0.1:8080',
'--proxy-bypass-list=localhost;127.0.0.1',
'--ignore-certificate-errors',
'--no-default-browser-check',
]
if self.config['host'] not in ('localhost', '127.0.0.1'):
chrome_args[0] = chrome_args[0].replace(
'127.0.0.1', self.config['host'])
if ' '.join(chrome_args) not in ' '.join(self.runner.cmdargs):
self.runner.cmdargs.extend(chrome_args)
self.runner.start()
proc = self.runner.process_handler
self.output_handler.proc = proc
self.control_server.browser_proc = proc
# pageload tests need to be able to access non-local connections via mitmproxy
if self.config['app'] == "firefox" and self.config['is_release_build'] and \
test.get('playback', None) is not None:
self.log.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=0")
os.environ['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = "0"
# if geckoProfile is enabled, initialize it
if self.config['gecko_profile'] is True:
self._init_gecko_profiling(test)
# tell the control server the gecko_profile dir; the control server will
# receive the actual gecko profiles from the web ext and will write them
# to disk; then profiles are picked up by gecko_profile.symbolicate
self.control_server.gecko_profile_dir = self.gecko_profiler.gecko_profile_dir
# set our cs flag to indicate we are running the browser/app
self.control_server._finished = False
# convert to seconds and account for page cycles
timeout = int(timeout / 1000) * int(test['page_cycles'])
# account for the pause the raptor webext runner takes after browser startup
timeout += int(self.post_startup_delay / 1000)
# if geckoProfile enabled, give browser more time for profiling
if self.config['gecko_profile'] is True:
timeout += 5 * 60
try:
elapsed_time = 0
while not self.control_server._finished:
time.sleep(1)
# we only want to force browser-shutdown on timeout if not in debug mode;
# in debug-mode we leave the browser running (require manual shutdown)
if not self.debug_mode:
elapsed_time += 1
if elapsed_time > (timeout) - 5: # stop 5 seconds early
self.log.info(
"application timed out after {} seconds".format(
timeout))
self.control_server.wait_for_quit()
break
finally:
if self.config['app'] == "geckoview":
if self.config['power_test']:
finish_geckoview_power_test(self)
self.check_for_crashes()
if self.playback is not None:
self.playback.stop()
# remove the raptor webext; as it must be reloaded with each subtest anyway
self.log.info("removing webext %s" % raptor_webext)
if self.config['app'] in ['firefox', 'geckoview', 'fennec']:
self.profile.addons.remove_addon(webext_id)
# for chrome the addon is just a list (appended to cmd line)
if self.config['app'] in ["chrome", "chrome-android"]:
self.profile.addons.remove(raptor_webext)
# gecko profiling symbolication
if self.config['gecko_profile'] is True:
self.gecko_profiler.symbolicate()
# clean up the temp gecko profiling folders
self.log.info("cleaning up after gecko profiling")
self.gecko_profiler.clean()
# browser should be closed by now but this is a backup-shutdown (if not in debug-mode)
if not self.debug_mode:
if self.config['app'] not in ['geckoview', 'fennec']:
if self.runner.is_running():
self.runner.stop()
# TODO the geckoview app should have been shutdown by this point by the
# control server, but we can double-check here to make sure
else:
# in debug mode, and running locally, leave the browser running
if self.config['run_local']:
self.log.info(
"* debug-mode enabled - please shutdown the browser manually..."
)
self.runner.wait(timeout=None)