def get_browser_meta(self):
"""Returns the browser name and version in a tuple (name, version).
Uses mozversion as the primary method to get this meta data and for
android this is the only method which exists to get this data. With android,
we use the installerpath attribute to determine this and this only works
with Firefox browsers.
"""
browser_name = None
browser_version = None
if self.config["app"] in self.firefox_android_apps:
try:
meta = mozversion.get_version(binary=self.installerpath)
browser_name = meta.get("application_name")
browser_version = meta.get("application_version")
except Exception as e:
LOG.warning(
"Failed to get android browser meta data through mozversion: %s-%s"
% (e.__class__.__name__, e)
)
if self.config["app"] == "chrome-m":
# We absolutely need to determine the chrome
# version here so that we can select the correct
# chromedriver for browsertime
from mozdevice import ADBDeviceFactory
device = ADBDeviceFactory(verbose=True)
binary = "com.android.chrome"
pkg_info = device.shell_output("dumpsys package %s" % binary)
version_matcher = re.compile(r".*versionName=([\d.]+)")
for line in pkg_info.split("\n"):
match = version_matcher.match(line)
if match:
browser_version = match.group(1)
browser_name = self.config["app"]
# First one found is the non-system
# or latest version.
break
if not browser_version:
raise Exception(
"Could not determine version for Google Chrome for Android"
)
if not browser_name:
LOG.warning("Could not find a browser name")
else:
LOG.info("Browser name: %s" % browser_name)
if not browser_version:
LOG.warning("Could not find a browser version")
else:
LOG.info("Browser version: %s" % browser_version)
return (browser_name, browser_version)
class RemoteCPPUnitTests(cppunittests.CPPUnitTests):
def __init__(self, options, progs):
cppunittests.CPPUnitTests.__init__(self)
self.options = options
self.device = ADBDeviceFactory(
adb=options.adb_path or "adb",
device=options.device_serial,
test_root=options.remote_test_root,
)
self.remote_test_root = posixpath.join(self.device.test_root,
"cppunittests")
self.remote_bin_dir = posixpath.join(self.remote_test_root, "b")
self.remote_tmp_dir = posixpath.join(self.remote_test_root, "tmp")
self.remote_home_dir = posixpath.join(self.remote_test_root, "h")
if options.setup:
self.setup_bin(progs)
def setup_bin(self, progs):
self.device.rm(self.remote_test_root, force=True, recursive=True)
self.device.mkdir(self.remote_home_dir, parents=True)
self.device.mkdir(self.remote_tmp_dir)
self.device.mkdir(self.remote_bin_dir)
self.push_libs()
self.push_progs(progs)
self.device.chmod(self.remote_bin_dir, recursive=True)
def push_libs(self):
if self.options.local_apk:
with mozfile.TemporaryDirectory() as tmpdir:
apk_contents = ZipFile(self.options.local_apk)
for info in apk_contents.infolist():
if info.filename.endswith(".so"):
print("Pushing %s.." % info.filename, file=sys.stderr)
remote_file = posixpath.join(
self.remote_bin_dir,
os.path.basename(info.filename))
apk_contents.extract(info, tmpdir)
local_file = os.path.join(tmpdir, info.filename)
with open(local_file) as f:
# Decompress xz-compressed file.
if f.read(5)[1:] == "7zXZ":
cmd = [
"xz", "-df", "--suffix", ".so", local_file
]
subprocess.check_output(cmd)
# xz strips the ".so" file suffix.
os.rename(local_file[:-3], local_file)
self.device.push(local_file, remote_file)
elif self.options.local_lib:
for path in os.listdir(self.options.local_lib):
if path.endswith(".so"):
print("Pushing {}..".format(path), file=sys.stderr)
remote_file = posixpath.join(self.remote_bin_dir, path)
local_file = os.path.join(self.options.local_lib, path)
self.device.push(local_file, remote_file)
# Additional libraries may be found in a sub-directory such as
# "lib/armeabi-v7a"
for subdir in ["assets", "lib"]:
local_arm_lib = os.path.join(self.options.local_lib, subdir)
if os.path.isdir(local_arm_lib):
for root, dirs, paths in os.walk(local_arm_lib):
for path in paths:
if path.endswith(".so"):
print("Pushing {}..".format(path),
file=sys.stderr)
remote_file = posixpath.join(
self.remote_bin_dir, path)
local_file = os.path.join(root, path)
self.device.push(local_file, remote_file)
def push_progs(self, progs):
for local_file in progs:
remote_file = posixpath.join(self.remote_bin_dir,
os.path.basename(local_file))
self.device.push(local_file, remote_file)
def build_environment(self, enable_webrender=False):
env = self.build_core_environment({}, enable_webrender)
env["LD_LIBRARY_PATH"] = self.remote_bin_dir
env["TMPDIR"] = self.remote_tmp_dir
env["HOME"] = self.remote_home_dir
env["MOZ_XRE_DIR"] = self.remote_bin_dir
if self.options.add_env:
for envdef in self.options.add_env:
envdef_parts = envdef.split("=", 1)
if len(envdef_parts) == 2:
env[envdef_parts[0]] = envdef_parts[1]
elif len(envdef_parts) == 1:
env[envdef_parts[0]] = ""
else:
self.log.warning("invalid --addEnv option skipped: %s" %
envdef)
return env
def run_one_test(self,
prog,
env,
symbols_path=None,
interactive=False,
timeout_factor=1):
"""
Run a single C++ unit test program remotely.
Arguments:
* prog: The path to the test program to run.
* env: The environment to use for running the program.
* symbols_path: A path to a directory containing Breakpad-formatted
symbol files for producing stack traces on crash.
* timeout_factor: An optional test-specific timeout multiplier.
Return True if the program exits with a zero status, False otherwise.
"""
basename = os.path.basename(prog)
remote_bin = posixpath.join(self.remote_bin_dir, basename)
self.log.test_start(basename)
test_timeout = cppunittests.CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor
try:
output = self.device.shell_output(remote_bin,
env=env,
cwd=self.remote_home_dir,
timeout=test_timeout)
returncode = 0
except ADBTimeoutError:
raise
except ADBProcessError as e:
output = e.adb_process.stdout
returncode = e.adb_process.exitcode
self.log.process_output(basename,
"\n%s" % output,
command=[remote_bin])
with mozfile.TemporaryDirectory() as tempdir:
self.device.pull(self.remote_home_dir, tempdir)
if mozcrash.check_for_crashes(tempdir,
symbols_path,
test_name=basename):
self.log.test_end(basename, status="CRASH", expected="PASS")
return False
result = returncode == 0
if not result:
self.log.test_end(
basename,
status="FAIL",
expected="PASS",
message=("test failed with return code %s" % returncode),
)
else:
self.log.test_end(basename, status="PASS", expected="PASS")
return result
class BrowsertimeAndroid(PerftestAndroid, Browsertime):
"""Android setup and configuration for browsertime
When running raptor-browsertime tests on android, we create the profile (and set the proxy
prefs in the profile that is using playback) but we don't need to copy it onto the device
because geckodriver takes care of that.
We tell browsertime to use our profile (we pass it in with the firefox.profileTemplate arg);
browsertime creates a copy of that and passes that into geckodriver. Geckodriver then takes
the profile and copies it onto the mobile device's test root for us; and then it even writes
the geckoview app config.yaml file onto the device, which points the app to the profile on
the device's test root.
Therefore, raptor doesn't have to copy the profile onto the scard (and create the config.yaml)
file ourselves. Also note when using playback, the nss certificate db is created as usual when
mitmproxy is started (and saved in the profile) so it is already included in the profile that
browsertime/geckodriver copies onto the device.
XXX: bc: This doesn't work with scoped storage in Android 10 since the shell owns the profile
directory that is pushed to the device and the profile can no longer be on the sdcard. But when
geckodriver's android.rs defines the profile to be located on internal storage, it will be
owned by shell but if we are attempting to eliminate root, then when we run shell commands
as the app, they will fail due to the app being unable to write to the shell owned profile
directory.
"""
def __init__(self, app, binary, activity=None, intent=None, **kwargs):
super(BrowsertimeAndroid, self).__init__(
app, binary, profile_class="firefox", **kwargs
)
self.config.update({"activity": activity, "intent": intent})
self.remote_test_root = None
self.remote_profile = None
@property
def browsertime_args(self):
if self.config['app'] == 'chrome-m':
args_list = [
'--browser', 'chrome',
'--android',
]
else:
activity = self.config["activity"]
if self.config["app"] == "fenix":
LOG.info(
"Changing initial activity to "
"`mozilla.telemetry.glean.debug.GleanDebugActivity`"
)
activity = "mozilla.telemetry.glean.debug.GleanDebugActivity"
args_list = [
"--browser", "firefox",
"--android",
# Work around a `selenium-webdriver` issue where Browsertime
# fails to find a Firefox binary even though we're going to
# actually do things on an Android device.
"--firefox.binaryPath", self.browsertime_node,
"--firefox.android.package", self.config["binary"],
"--firefox.android.activity", activity,
]
# if running on Fenix we must add the intent as we use a special non-default one there
if self.config["app"] == "fenix" and self.config.get("intent") is not None:
args_list.extend(["--firefox.android.intentArgument=-a"])
args_list.extend(
["--firefox.android.intentArgument", self.config["intent"]]
)
# Change glean ping names in all cases on Fenix
args_list.extend([
"--firefox.android.intentArgument=--es",
"--firefox.android.intentArgument=startNext",
"--firefox.android.intentArgument=" + self.config["activity"],
"--firefox.android.intentArgument=--esa",
"--firefox.android.intentArgument=sourceTags",
"--firefox.android.intentArgument=automation",
])
args_list.extend(["--firefox.android.intentArgument=-d"])
args_list.extend(["--firefox.android.intentArgument", str("about:blank")])
return args_list
def setup_chrome_args(self, test):
chrome_args = ["--use-mock-keychain", "--no-default-browser-check", "--no-first-run"]
if test.get("playback", False):
pb_args = [
"--proxy-server=%s:%d" % (self.playback.host, self.playback.port),
"--proxy-bypass-list=localhost;127.0.0.1",
"--ignore-certificate-errors",
]
if not self.is_localhost:
pb_args[0] = pb_args[0].replace("127.0.0.1", self.config["host"])
chrome_args.extend(pb_args)
if self.debug_mode:
chrome_args.extend(["--auto-open-devtools-for-tabs"])
args_list = []
for arg in chrome_args:
args_list.extend(["--chrome.args=" + str(arg.replace("'", '"'))])
return args_list
def build_browser_profile(self):
super(BrowsertimeAndroid, self).build_browser_profile()
# Merge in the Android profile.
path = os.path.join(self.profile_data_dir, "raptor-android")
LOG.info("Merging profile: {}".format(path))
self.profile.merge(path)
self.profile.set_preferences(
{"browser.tabs.remote.autostart": self.config["e10s"]}
)
# There's no great way to have "after" advice in Python, so we do this
# in super and then again here since the profile merging re-introduces
# the "#MozRunner" delimiters.
self.remove_mozprofile_delimiters_from_profile()
def setup_adb_device(self):
if self.device is None:
self.device = ADBDeviceFactory(verbose=True)
if not self.config.get("disable_perf_tuning", False):
tune_performance(self.device, log=LOG)
self.clear_app_data()
self.set_debug_app_flag()
self.device.run_as_package = self.config['binary']
self.remote_test_root = os.path.join(self.device.test_root, "raptor")
self.remote_profile = os.path.join(self.remote_test_root, "profile")
def run_test_setup(self, test):
super(BrowsertimeAndroid, self).run_test_setup(test)
self.set_reverse_ports()
if self.playback:
self.turn_on_android_app_proxy()
self.remove_mozprofile_delimiters_from_profile()
def run_tests(self, tests, test_names):
self.setup_adb_device()
if self.config['app'] == "chrome-m":
# Make sure that chrome is enabled on the device
self.device.shell_output("pm enable com.android.chrome")
return super(BrowsertimeAndroid, self).run_tests(tests, test_names)
def run_test_teardown(self, test):
LOG.info("removing reverse socket connections")
self.device.remove_socket_connections("reverse")
super(BrowsertimeAndroid, self).run_test_teardown(test)
class WebExtensionAndroid(PerftestAndroid, WebExtension):
def __init__(self, app, binary, activity=None, intent=None, **kwargs):
super(WebExtensionAndroid, self).__init__(app,
binary,
profile_class="firefox",
**kwargs)
self.config.update({"activity": activity, "intent": intent})
self.os_baseline_data = None
self.power_test_time = None
self.screen_off_timeout = 0
self.screen_brightness = 127
self.app_launched = False
def setup_adb_device(self):
if self.device is None:
self.device = ADBDeviceFactory(verbose=True)
if not self.config.get("disable_perf_tuning", False):
tune_performance(self.device, log=LOG)
self.device.run_as_package = self.config["binary"]
self.remote_test_root = os.path.join(self.device.test_root, "raptor")
self.remote_profile = os.path.join(self.remote_test_root, "profile")
if self.config["power_test"]:
disable_charging(self.device)
LOG.info("creating remote root folder for raptor: %s" %
self.remote_test_root)
self.device.rm(self.remote_test_root, force=True, recursive=True)
self.device.mkdir(self.remote_test_root, parents=True)
self.clear_app_data()
self.set_debug_app_flag()
def process_exists(self):
return self.device is not None and self.device.process_exist(
self.config["binary"])
def write_android_app_config(self):
# geckoview supports having a local on-device config file; use this file
# to tell the app to use the specified browser profile, as well as other opts
# on-device: /data/local/tmp/com.yourcompany.yourapp-geckoview-config.yaml
# https://mozilla.github.io/geckoview/tutorials/automation.html#configuration-file-format
# only supported for geckoview apps
if self.config["app"] == "fennec":
return
LOG.info("creating android app config.yml")
yml_config_data = dict(
args=[
"--profile",
self.remote_profile,
"--allow-downgrade",
],
env=dict(
LOG_VERBOSE=1,
R_LOG_LEVEL=6,
MOZ_WEBRENDER=int(self.config["enable_webrender"]),
),
)
yml_name = "%s-geckoview-config.yaml" % self.config["binary"]
yml_on_host = os.path.join(tempfile.mkdtemp(), yml_name)
write_yml_file(yml_on_host, yml_config_data)
yml_on_device = os.path.join("/data", "local", "tmp", yml_name)
try:
LOG.info("copying %s to device: %s" % (yml_on_host, yml_on_device))
self.device.rm(yml_on_device, force=True, recursive=True)
self.device.push(yml_on_host, yml_on_device)
except Exception:
LOG.critical("failed to push %s to device!" % yml_on_device)
raise
def log_android_device_temperature(self):
# retrieve and log the android device temperature
try:
# use sort since cat gives I/O Error on Pixel 2 - 10.
thermal_zone0 = self.device.shell_output(
"sort /sys/class/thermal/thermal_zone0/temp")
try:
thermal_zone0 = "%.3f" % (float(thermal_zone0) / 1000)
except ValueError:
thermal_zone0 = "Unknown"
except ADBProcessError:
thermal_zone0 = "Unknown"
try:
zone_type = self.device.shell_output(
"cat /sys/class/thermal/thermal_zone0/type")
except ADBProcessError:
zone_type = "Unknown"
LOG.info("(thermal_zone0) device temperature: %s zone type: %s" %
(thermal_zone0, zone_type))
def launch_firefox_android_app(self, test_name):
LOG.info("starting %s" % self.config["app"])
extra_args = [
"-profile",
self.remote_profile,
"--allow-downgrade",
"--es",
"env0",
"LOG_VERBOSE=1",
"--es",
"env1",
"R_LOG_LEVEL=6",
"--es",
"env2",
"MOZ_WEBRENDER=%d" % self.config["enable_webrender"],
# Force the app to immediately exit for content crashes
"--es",
"env3",
"MOZ_CRASHREPORTER_SHUTDOWN=1",
]
try:
# make sure the android app is not already running
self.device.stop_application(self.config["binary"])
if self.config["app"] == "fennec":
self.device.launch_fennec(
self.config["binary"],
extra_args=extra_args,
url="about:blank",
fail_if_running=False,
)
else:
# command line 'extra' args not used with geckoview apps; instead we use
# an on-device config.yml file (see write_android_app_config)
self.device.launch_application(
self.config["binary"],
self.config["activity"],
self.config["intent"],
extras=None,
url="about:blank",
fail_if_running=False,
)
# Check if app has started and it's running
if not self.process_exists:
raise Exception(
"Error launching %s. App did not start properly!" %
self.config["binary"])
self.app_launched = True
except Exception as e:
LOG.error("Exception launching %s" % self.config["binary"])
LOG.error("Exception: %s %s" % (type(e).__name__, str(e)))
if self.config["power_test"]:
finish_android_power_test(self, test_name)
raise
# give our control server the device and app info
self.control_server.device = self.device
self.control_server.app_name = self.config["binary"]
def copy_cert_db(self, source_dir, target_dir):
# copy browser cert db (that was previously created via certutil) from source to target
cert_db_files = ["pkcs11.txt", "key4.db", "cert9.db"]
for next_file in cert_db_files:
_source = os.path.join(source_dir, next_file)
_dest = os.path.join(target_dir, next_file)
if os.path.exists(_source):
LOG.info("copying %s to %s" % (_source, _dest))
shutil.copyfile(_source, _dest)
else:
LOG.critical("unable to find ssl cert db file: %s" % _source)
def run_tests(self, tests, test_names):
self.setup_adb_device()
return super(WebExtensionAndroid, self).run_tests(tests, test_names)
def run_test_setup(self, test):
super(WebExtensionAndroid, self).run_test_setup(test)
self.set_reverse_ports()
def run_test_teardown(self, test):
LOG.info("removing reverse socket connections")
self.device.remove_socket_connections("reverse")
super(WebExtensionAndroid, self).run_test_teardown(test)
def run_test(self, test, timeout):
# tests will be run warm (i.e. NO browser restart between page-cycles)
# unless otheriwse specified in the test INI by using 'cold = true'
try:
if self.config["power_test"]:
# gather OS baseline data
init_android_power_test(self)
LOG.info("Running OS baseline, pausing for 1 minute...")
time.sleep(60)
LOG.info("Finishing baseline...")
finish_android_power_test(self,
"os-baseline",
os_baseline=True)
# initialize for the test
init_android_power_test(self)
if self.config.get("cold") or test.get("cold"):
self.__run_test_cold(test, timeout)
else:
self.__run_test_warm(test, timeout)
except SignalHandlerException:
self.device.stop_application(self.config["binary"])
if self.config["power_test"]:
enable_charging(self.device)
finally:
if self.config["power_test"]:
finish_android_power_test(self, test["name"])
def __run_test_cold(self, test, timeout):
"""
Run the Raptor test but restart the entire browser app between page-cycles.
Note: For page-load tests, playback will only be started once - at the beginning of all
browser cycles, and then stopped after all cycles are finished. The proxy is set via prefs
in the browser profile so those will need to be set again in each new profile/cycle.
Note that instead of using the certutil tool each time to create a db and import the
mitmproxy SSL cert (it's done in mozbase/mozproxy) we will simply copy the existing
cert db from the first cycle's browser profile into the new clean profile; this way
we don't have to re-create the cert db on each browser cycle.
Since we're running in cold-mode, before this point (in manifest.py) the
'expected-browser-cycles' value was already set to the initial 'page-cycles' value;
and the 'page-cycles' value was set to 1 as we want to perform one page-cycle per
browser restart.
The 'browser-cycle' value is the current overall browser start iteration. The control
server will receive the current 'browser-cycle' and the 'expected-browser-cycles' in
each results set received; and will pass that on as part of the results so that the
results processing will know results for multiple browser cycles are being received.
The default will be to run in warm mode; unless 'cold = true' is set in the test INI.
"""
LOG.info(
"test %s is running in cold mode; browser WILL be restarted between "
"page cycles" % test["name"])
for test["browser_cycle"] in range(1, test["expected_browser_cycles"] +
1):
LOG.info("begin browser cycle %d of %d for test %s" %
(test["browser_cycle"], test["expected_browser_cycles"],
test["name"]))
self.run_test_setup(test)
self.clear_app_data()
self.set_debug_app_flag()
if test["browser_cycle"] == 1:
if test.get("playback") is not None:
# an ssl cert db has now been created in the profile; copy it out so we
# can use the same cert db in future test cycles / browser restarts
local_cert_db_dir = tempfile.mkdtemp()
LOG.info(
"backing up browser ssl cert db that was created via certutil"
)
self.copy_cert_db(self.config["local_profile_dir"],
local_cert_db_dir)
if not self.is_localhost:
self.delete_proxy_settings_from_profile()
else:
# double-check to ensure app has been shutdown
self.device.stop_application(self.config["binary"])
# initial browser profile was already created before run_test was called;
# now additional browser cycles we want to create a new one each time
self.build_browser_profile()
if test.get("playback") is not None:
# get cert db from previous cycle profile and copy into new clean profile
# this saves us from having to start playback again / recreate cert db etc.
LOG.info(
"copying existing ssl cert db into new browser profile"
)
self.copy_cert_db(local_cert_db_dir,
self.config["local_profile_dir"])
self.run_test_setup(test)
if test.get("playback") is not None:
self.turn_on_android_app_proxy()
self.copy_profile_to_device()
self.log_android_device_temperature()
# write android app config.yml
self.write_android_app_config()
# now start the browser/app under test
self.launch_firefox_android_app(test["name"])
# set our control server flag to indicate we are running the browser/app
self.control_server._finished = False
if self.config["cpu_test"]:
# start measuring CPU usage
self.cpu_profiler = start_android_cpu_profiler(self)
self.wait_for_test_finish(test, timeout, self.process_exists)
# in debug mode, and running locally, leave the browser running
if self.debug_mode and self.config["run_local"]:
LOG.info(
"* debug-mode enabled - please shutdown the browser manually..."
)
self.runner.wait(timeout=None)
# break test execution if a exception is present
if len(self.results_handler.page_timeout_list) > 0:
break
def __run_test_warm(self, test, timeout):
LOG.info(
"test %s is running in warm mode; browser will NOT be restarted between "
"page cycles" % test["name"])
self.run_test_setup(test)
if not self.is_localhost:
self.delete_proxy_settings_from_profile()
if test.get("playback") is not None:
self.turn_on_android_app_proxy()
self.clear_app_data()
self.set_debug_app_flag()
self.copy_profile_to_device()
self.log_android_device_temperature()
# write android app config.yml
self.write_android_app_config()
# now start the browser/app under test
self.launch_firefox_android_app(test["name"])
# set our control server flag to indicate we are running the browser/app
self.control_server._finished = False
if self.config["cpu_test"]:
# start measuring CPU usage
self.cpu_profiler = start_android_cpu_profiler(self)
self.wait_for_test_finish(test, timeout, self.process_exists)
# in debug mode, and running locally, leave the browser running
if self.debug_mode and self.config["run_local"]:
LOG.info(
"* debug-mode enabled - please shutdown the browser manually..."
)
def check_for_crashes(self):
super(WebExtensionAndroid, self).check_for_crashes()
if not self.app_launched:
LOG.info(
"skipping check_for_crashes: application has not been launched"
)
return
self.app_launched = False
try:
dump_dir = tempfile.mkdtemp()
remote_dir = posixpath.join(self.remote_profile, "minidumps")
if not self.device.is_dir(remote_dir):
return
self.device.pull(remote_dir, dump_dir)
self.crashes += mozcrash.log_crashes(LOG, dump_dir,
self.config["symbols_path"])
finally:
try:
shutil.rmtree(dump_dir)
except Exception:
LOG.warning("unable to remove directory: %s" % dump_dir)
def clean_up(self):
LOG.info("removing test folder for raptor: %s" % self.remote_test_root)
self.device.rm(self.remote_test_root, force=True, recursive=True)
if self.config["power_test"]:
enable_charging(self.device)
super(WebExtensionAndroid, self).clean_up()