def run_tests_remote(tests, num_tests, prefix, options, slog):
# Setup device with everything needed to run our tests.
from mozdevice import ADBAndroid
device = ADBAndroid(device=options.device_serial,
test_root=options.remote_test_root)
# Update the test root to point to our test directory.
jit_tests_dir = posixpath.join(options.remote_test_root, 'jit-tests')
options.remote_test_root = posixpath.join(jit_tests_dir, 'jit-tests')
# Push js shell and libraries.
device.rm(jit_tests_dir, force=True, recursive=True)
device.mkdir(options.remote_test_root, parents=True)
push_libs(options, device)
push_progs(options, device, [prefix[0]])
device.chmod(options.remote_test_root, recursive=True, root=True)
JitTest.CacheDir = posixpath.join(options.remote_test_root, '.js-cache')
device.mkdir(JitTest.CacheDir)
device.push(JS_TESTS_DIR,
posixpath.join(jit_tests_dir, 'tests'),
timeout=600)
device.push(os.path.dirname(TEST_DIR),
options.remote_test_root,
timeout=600)
prefix[0] = os.path.join(options.remote_test_root, 'js')
# Run all tests.
pb = create_progressbar(num_tests, options)
gen = get_remote_results(tests, device, prefix, options)
ok = process_test_results(gen, num_tests, pb, options, slog)
return ok
class Raptor(object):
"""Container class for Raptor"""
def __init__(self, app, binary, run_local=False, obj_path=None):
self.config = {}
self.config['app'] = app
self.config['binary'] = binary
self.config['platform'] = mozinfo.os
self.config['processor'] = mozinfo.processor
self.config['run_local'] = run_local
self.config['obj_path'] = obj_path
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
self.log = get_default_logger(component='raptor-main')
self.control_server = None
self.playback = None
self.benchmark = None
# Create the profile; for geckoview we want a firefox profile type
if self.config['app'] == 'geckoview':
self.profile = create_profile('firefox')
else:
self.profile = create_profile(self.config['app'])
# Merge in base profiles
with open(os.path.join(self.profile_data_dir, 'profiles.json'),
'r') as fh:
base_profiles = json.load(fh)['raptor']
for name in base_profiles:
path = os.path.join(self.profile_data_dir, name)
self.log.info("Merging profile: {}".format(path))
self.profile.merge(path)
# create results holder
self.results_handler = RaptorResultsHandler()
# when testing desktop browsers we use mozrunner to start the browser; when
# testing on android (i.e. geckoview) we use mozdevice to control the device app
if self.config['app'] == "geckoview":
# create the android device handler; it gets initiated and sets up adb etc
self.log.info("creating android device handler using mozdevice")
self.device = ADBAndroid(verbose=True)
self.device.clear_logcat()
else:
# create the desktop browser runner
self.log.info("creating browser runner using mozrunner")
self.output_handler = OutputHandler()
process_args = {
'processOutputLine': [self.output_handler],
}
runner_cls = runners[app]
self.runner = runner_cls(binary,
profile=self.profile,
process_args=process_args)
self.log.info("raptor config: %s" % str(self.config))
@property
def profile_data_dir(self):
if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
'testing', 'profiles')
if build:
return os.path.join(build.topsrcdir, 'testing', 'profiles')
return os.path.join(here, 'profile_data')
def start_control_server(self):
self.control_server = RaptorControlServer(self.results_handler)
self.control_server.start()
# for android we must make the control server available to the device
if self.config['app'] == "geckoview":
self.log.info(
"making the raptor control server port available to device")
_tcp_port = "tcp:%s" % self.control_server.port
self.device.create_socket_connection('reverse', _tcp_port,
_tcp_port)
def get_playback_config(self, test):
self.config['playback_tool'] = test.get('playback')
self.log.info("test uses playback tool: %s " %
self.config['playback_tool'])
self.config['playback_binary_manifest'] = test.get(
'playback_binary_manifest', None)
_key = 'playback_binary_zip_%s' % self.config['platform']
self.config['playback_binary_zip'] = test.get(_key, None)
self.config['playback_pageset_manifest'] = test.get(
'playback_pageset_manifest', None)
_key = 'playback_pageset_zip_%s' % self.config['platform']
self.config['playback_pageset_zip'] = test.get(_key, None)
self.config['playback_recordings'] = test.get('playback_recordings',
None)
self.config['python3_win_manifest'] = test.get('python3_win_manifest',
None)
def run_test(self, test, timeout=None):
self.log.info("starting raptor test: %s" % test['name'])
self.log.info("test settings: %s" % str(test))
self.log.info("raptor config: %s" % str(self.config))
# benchmark-type tests require the benchmark test to be served out
if test.get('type') == "benchmark":
self.benchmark = Benchmark(self.config, test)
benchmark_port = int(self.benchmark.port)
else:
benchmark_port = 0
gen_test_config(self.config['app'], test['name'],
self.control_server.port, benchmark_port)
# for android we must make the benchmarks server available to the device
if self.config['app'] == "geckoview":
self.log.info(
"making the raptor benchmarks server port available to device")
_tcp_port = "tcp:%s" % benchmark_port
self.device.create_socket_connection('reverse', _tcp_port,
_tcp_port)
# must intall raptor addon each time because we dynamically update some content
raptor_webext = os.path.join(webext_dir, 'raptor')
self.log.info("installing webext %s" % raptor_webext)
self.profile.addons.install(raptor_webext)
# add test specific preferences
if test.get("preferences", None) is not None:
if self.config['app'] == "firefox":
self.profile.set_preferences(json.loads(test['preferences']))
else:
self.log.info("preferences were configured for the test, \
but we do not install them on non Firefox browsers."
)
# on firefox we can get an addon id; chrome addon actually is just cmd line arg
if self.config['app'] in ["firefox", "geckoview"]:
webext_id = self.profile.addons.addon_details(raptor_webext)['id']
# some tests require tools to playback the test pages
if test.get('playback', None) is not None:
self.get_playback_config(test)
# startup the playback tool
self.playback = get_playback(self.config)
# for geckoview we must copy the profile onto the device and set perms
if self.config['app'] == "geckoview":
if not self.device.is_app_installed(self.config['binary']):
raise Exception('%s is not installed' % self.config['binary'])
self.log.info("copying firefox profile onto the android device")
self.device_profile = "/sdcard/raptor-profile"
if self.device.is_dir(self.device_profile):
self.device.rm(self.device_profile, recursive=True)
self.device.mkdir(self.device_profile)
self.device.push(self.profile.profile, self.device_profile)
self.log.info("setting permisions to profile dir on the device")
self.device.chmod(self.device_profile, recursive=True)
# now start the geckoview app
self.log.info("starting %s" % self.config['app'])
extra_args = [
"-profile", self.device_profile, "--es", "env0",
"LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6"
]
try:
# make sure the geckoview app is not running before
# attempting to start.
self.device.stop_application(self.config['binary'])
self.device.launch_activity(self.config['binary'],
"GeckoViewActivity",
extra_args=extra_args,
url='about:blank',
fail_if_running=False)
except Exception:
self.log.error("Exception launching %s" %
self.config['binary'])
raise
self.control_server.device = self.device
self.control_server.app_name = self.config['binary']
else:
# now start the desktop browser
self.log.info("starting %s" % self.config['app'])
self.runner.start()
proc = self.runner.process_handler
self.output_handler.proc = proc
self.control_server.browser_proc = proc
# set our cs flag to indicate we are running the browser/app
self.control_server._finished = False
# convert to seconds and account for page cycles
timeout = int(timeout / 1000) * int(test['page_cycles'])
try:
elapsed_time = 0
while not self.control_server._finished:
time.sleep(1)
elapsed_time += 1
if elapsed_time > (timeout) - 5: # stop 5 seconds early
self.log.info(
"application timed out after {} seconds".format(
timeout))
self.control_server.wait_for_quit()
break
finally:
if self.config['app'] != "geckoview":
try:
self.runner.check_for_crashes()
except NotImplementedError: # not implemented for Chrome
pass
# TODO: if on geckoview is there some cleanup here i.e. check for crashes?
if self.playback is not None:
self.playback.stop()
# remove the raptor webext; as it must be reloaded with each subtest anyway
# applies to firefox only; chrome the addon is actually just cmd line arg
if self.config['app'] in ["firefox", "geckoview"]:
self.log.info("removing webext %s" % raptor_webext)
self.profile.addons.remove_addon(webext_id)
if self.config['app'] != "geckoview":
if self.runner.is_running():
self.runner.stop()
# TODO the geckoview app should have been shutdown by this point by the
# control server, but we can double-check here to make sure
def process_results(self):
# when running locally output results in build/raptor.json; when running
# in production output to a local.json to be turned into tc job artifact
if self.config.get('run_local', False):
if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
raptor_json_path = os.path.join(
os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing',
'mozharness', 'build', 'raptor.json')
else:
raptor_json_path = os.path.join(here, 'raptor.json')
else:
raptor_json_path = os.path.join(os.getcwd(), 'local.json')
self.config['raptor_json_path'] = raptor_json_path
return self.results_handler.summarize_and_output(self.config)
def clean_up(self):
self.control_server.stop()
if self.config['app'] != "geckoview":
self.runner.stop()
elif self.config['app'] == 'geckoview':
self.log.info('removing reverse socket connections')
self.device.remove_socket_connections('reverse')
else:
pass
self.log.info("finished")
class RemoteReftest(RefTest):
use_marionette = False
resolver_cls = RemoteReftestResolver
def __init__(self, options, scriptDir):
RefTest.__init__(self, options.suite)
self.run_by_manifest = False
self.scriptDir = scriptDir
self.localLogName = options.localLogName
verbose = False
if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug':
verbose = True
print "set verbose!"
self.device = ADBAndroid(adb=options.adb_path,
device=options.deviceSerial,
test_root=options.remoteTestRoot,
verbose=verbose)
if options.remoteTestRoot is None:
options.remoteTestRoot = posixpath.join(self.device.test_root,
"reftest")
options.remoteProfile = posixpath.join(options.remoteTestRoot,
"profile")
options.remoteLogFile = posixpath.join(options.remoteTestRoot,
"reftest.log")
options.logFile = options.remoteLogFile
self.remoteProfile = options.remoteProfile
self.remoteTestRoot = options.remoteTestRoot
if not options.ignoreWindowSize:
parts = self.device.get_info('screen')['screen'][0].split()
width = int(parts[0].split(':')[1])
height = int(parts[1].split(':')[1])
if (width < 1366 or height < 1050):
self.error("ERROR: Invalid screen resolution %sx%s, "
"please adjust to 1366x1050 or higher" %
(width, height))
self._populate_logger(options)
self.outputHandler = OutputHandler(self.log, options.utilityPath,
options.symbolsPath)
# RemoteAutomation.py's 'messageLogger' is also used by mochitest. Mimic a mochitest
# MessageLogger object to re-use this code path.
self.outputHandler.write = self.outputHandler.__call__
self.automation = RemoteAutomation(self.device,
options.app,
self.remoteProfile,
options.remoteLogFile,
processArgs=None)
self.automation._processArgs['messageLogger'] = self.outputHandler
self.environment = self.automation.environment
if self.automation.IS_DEBUG_BUILD:
self.SERVER_STARTUP_TIMEOUT = 180
else:
self.SERVER_STARTUP_TIMEOUT = 90
self.remoteCache = os.path.join(options.remoteTestRoot, "cache/")
# Check that Firefox is installed
expected = options.app.split('/')[-1]
if not self.device.is_app_installed(expected):
raise Exception("%s is not installed on this device" % expected)
self.automation.deleteANRs()
self.automation.deleteTombstones()
self.device.clear_logcat()
self.device.rm(self.remoteCache, force=True, recursive=True)
procName = options.app.split('/')[-1]
self.device.pkill(procName)
if self.device.process_exist(procName):
self.log.error("unable to kill %s before starting tests!" %
procName)
def findPath(self, paths, filename=None):
for path in paths:
p = path
if filename:
p = os.path.join(p, filename)
if os.path.exists(self.getFullPath(p)):
return path
return None
def startWebServer(self, options):
""" Create the webserver on the host and start it up """
remoteXrePath = options.xrePath
remoteUtilityPath = options.utilityPath
localAutomation = Automation()
localAutomation.IS_WIN32 = False
localAutomation.IS_LINUX = False
localAutomation.IS_MAC = False
localAutomation.UNIXISH = False
hostos = sys.platform
if (hostos == 'mac' or hostos == 'darwin'):
localAutomation.IS_MAC = True
elif (hostos == 'linux' or hostos == 'linux2'):
localAutomation.IS_LINUX = True
localAutomation.UNIXISH = True
elif (hostos == 'win32' or hostos == 'win64'):
localAutomation.BIN_SUFFIX = ".exe"
localAutomation.IS_WIN32 = True
paths = [options.xrePath, localAutomation.DIST_BIN]
options.xrePath = self.findPath(paths)
if options.xrePath is None:
print(
"ERROR: unable to find xulrunner path for %s, "
"please specify with --xre-path" % (os.name))
return 1
paths.append("bin")
paths.append(os.path.join("..", "bin"))
xpcshell = "xpcshell"
if (os.name == "nt"):
xpcshell += ".exe"
if (options.utilityPath):
paths.insert(0, options.utilityPath)
options.utilityPath = self.findPath(paths, xpcshell)
if options.utilityPath is None:
print(
"ERROR: unable to find utility path for %s, "
"please specify with --utility-path" % (os.name))
return 1
options.serverProfilePath = tempfile.mkdtemp()
self.server = ReftestServer(localAutomation, options, self.scriptDir)
retVal = self.server.start()
if retVal:
return retVal
retVal = self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT)
if retVal:
return retVal
options.xrePath = remoteXrePath
options.utilityPath = remoteUtilityPath
return 0
def stopWebServer(self, options):
self.server.stop()
def killNamedProc(self, pname, orphans=True):
""" Kill processes matching the given command name """
self.log.info("Checking for %s processes..." % pname)
for proc in psutil.process_iter():
try:
if proc.name() == pname:
procd = proc.as_dict(
attrs=['pid', 'ppid', 'name', 'username'])
if proc.ppid() == 1 or not orphans:
self.log.info("killing %s" % procd)
try:
os.kill(proc.pid,
getattr(signal, "SIGKILL", signal.SIGTERM))
except Exception as e:
self.log.info("Failed to kill process %d: %s" %
(proc.pid, str(e)))
else:
self.log.info("NOT killing %s (not an orphan?)" %
procd)
except Exception:
# may not be able to access process info for all processes
continue
def createReftestProfile(self, options, **kwargs):
profile = RefTest.createReftestProfile(self,
options,
server=options.remoteWebServer,
port=options.httpPort,
**kwargs)
profileDir = profile.profile
prefs = {}
prefs["app.update.url.android"] = ""
prefs["browser.firstrun.show.localepicker"] = False
prefs["reftest.remote"] = True
prefs[
"datareporting.policy.dataSubmissionPolicyBypassAcceptance"] = True
# move necko cache to a location that can be cleaned up
prefs["browser.cache.disk.parent_directory"] = self.remoteCache
prefs["layout.css.devPixelsPerPx"] = "1.0"
# Because Fennec is a little wacky (see bug 1156817) we need to load the
# reftest pages at 1.0 zoom, rather than zooming to fit the CSS viewport.
prefs["apz.allow_zooming"] = False
# Set the extra prefs.
profile.set_preferences(prefs)
try:
self.device.push(profileDir, options.remoteProfile)
self.device.chmod(options.remoteProfile, recursive=True)
except Exception:
print "Automation Error: Failed to copy profiledir to device"
raise
return profile
def copyExtraFilesToProfile(self, options, profile):
profileDir = profile.profile
RefTest.copyExtraFilesToProfile(self, options, profile)
if len(os.listdir(profileDir)) > 0:
try:
self.device.push(profileDir, options.remoteProfile)
self.device.chmod(options.remoteProfile, recursive=True)
except Exception:
print "Automation Error: Failed to copy extra files to device"
raise
def printDeviceInfo(self, printLogcat=False):
try:
if printLogcat:
logcat = self.device.get_logcat(
filter_out_regexps=fennecLogcatFilters)
print ''.join(logcat)
print "Device info:"
devinfo = self.device.get_info()
for category in devinfo:
if type(devinfo[category]) is list:
print " %s:" % category
for item in devinfo[category]:
print " %s" % item
else:
print " %s: %s" % (category, devinfo[category])
print "Test root: %s" % self.device.test_root
except Exception as e:
print "WARNING: Error getting device information: %s" % str(e)
def environment(self, **kwargs):
return self.automation.environment(**kwargs)
def buildBrowserEnv(self, options, profileDir):
browserEnv = RefTest.buildBrowserEnv(self, options, profileDir)
# remove desktop environment not used on device
if "XPCOM_MEM_BLOAT_LOG" in browserEnv:
del browserEnv["XPCOM_MEM_BLOAT_LOG"]
return browserEnv
def runApp(self,
options,
cmdargs=None,
timeout=None,
debuggerInfo=None,
symbolsPath=None,
valgrindPath=None,
valgrindArgs=None,
valgrindSuppFiles=None,
**profileArgs):
if cmdargs is None:
cmdargs = []
if self.use_marionette:
cmdargs.append('-marionette')
binary = options.app
profile = self.createReftestProfile(options, **profileArgs)
# browser environment
env = self.buildBrowserEnv(options, profile.profile)
self.log.info("Running with e10s: {}".format(options.e10s))
status, self.lastTestSeen = self.automation.runApp(
None,
env,
binary,
profile.profile,
cmdargs,
utilityPath=options.utilityPath,
xrePath=options.xrePath,
debuggerInfo=debuggerInfo,
symbolsPath=symbolsPath,
timeout=timeout)
self.cleanup(profile.profile)
return status
def cleanup(self, profileDir):
self.device.rm(self.remoteTestRoot, force=True, recursive=True)
self.device.rm(self.remoteProfile, force=True, recursive=True)
self.device.rm(self.remoteCache, force=True, recursive=True)
RefTest.cleanup(self, profileDir)
class MochiRemote(MochitestDesktop):
localProfile = None
logMessages = []
def __init__(self, options):
MochitestDesktop.__init__(self, options.flavor, vars(options))
verbose = False
if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug':
verbose = True
if hasattr(options, 'log'):
delattr(options, 'log')
self.certdbNew = True
self.chromePushed = False
self.mozLogName = "moz.log"
self.device = ADBAndroid(adb=options.adbPath,
device=options.deviceSerial,
test_root=options.remoteTestRoot,
verbose=verbose)
if options.remoteTestRoot is None:
options.remoteTestRoot = self.device.test_root
options.dumpOutputDirectory = options.remoteTestRoot
self.remoteLogFile = posixpath.join(options.remoteTestRoot, "logs",
"mochitest.log")
logParent = posixpath.dirname(self.remoteLogFile)
self.device.rm(logParent, force=True, recursive=True)
self.device.mkdir(logParent)
self.remoteProfile = posixpath.join(options.remoteTestRoot, "profile/")
self.device.rm(self.remoteProfile, force=True, recursive=True)
self.counts = dict()
self.message_logger = MessageLogger(logger=None)
self.message_logger.logger = self.log
process_args = {
'messageLogger': self.message_logger,
'counts': self.counts
}
self.automation = RemoteAutomation(self.device,
options.remoteappname,
self.remoteProfile,
self.remoteLogFile,
processArgs=process_args)
self.environment = self.automation.environment
# Check that Firefox is installed
expected = options.app.split('/')[-1]
if not self.device.is_app_installed(expected):
raise Exception("%s is not installed on this device" % expected)
self.automation.deleteANRs()
self.automation.deleteTombstones()
self.device.clear_logcat()
self.remoteModulesDir = posixpath.join(options.remoteTestRoot,
"modules/")
self.remoteCache = posixpath.join(options.remoteTestRoot, "cache/")
self.device.rm(self.remoteCache, force=True, recursive=True)
# move necko cache to a location that can be cleaned up
options.extraPrefs += [
"browser.cache.disk.parent_directory=%s" % self.remoteCache
]
self.remoteMozLog = posixpath.join(options.remoteTestRoot, "mozlog")
self.device.rm(self.remoteMozLog, force=True, recursive=True)
self.device.mkdir(self.remoteMozLog)
self.remoteChromeTestDir = posixpath.join(options.remoteTestRoot,
"chrome")
self.device.rm(self.remoteChromeTestDir, force=True, recursive=True)
self.device.mkdir(self.remoteChromeTestDir)
procName = options.app.split('/')[-1]
self.device.stop_application(procName)
if self.device.process_exist(procName):
self.log.warning("unable to kill %s before running tests!" %
procName)
# Add Android version (SDK level) to mozinfo so that manifest entries
# can be conditional on android_version.
self.log.info(
"Android sdk version '%s'; will use this to filter manifests" %
str(self.device.version))
mozinfo.info['android_version'] = str(self.device.version)
def cleanup(self, options, final=False):
if final:
self.device.rm(self.remoteChromeTestDir,
force=True,
recursive=True)
self.chromePushed = False
uploadDir = os.environ.get('MOZ_UPLOAD_DIR', None)
if uploadDir and self.device.is_dir(self.remoteMozLog):
self.device.pull(self.remoteMozLog, uploadDir)
self.device.rm(self.remoteLogFile, force=True)
self.device.rm(self.remoteProfile, force=True, recursive=True)
self.device.rm(self.remoteCache, force=True, recursive=True)
MochitestDesktop.cleanup(self, options, final)
self.localProfile = None
def findPath(self, paths, filename=None):
for path in paths:
p = path
if filename:
p = os.path.join(p, filename)
if os.path.exists(self.getFullPath(p)):
return path
return None
def makeLocalAutomation(self):
localAutomation = Automation()
localAutomation.IS_WIN32 = False
localAutomation.IS_LINUX = False
localAutomation.IS_MAC = False
localAutomation.UNIXISH = False
hostos = sys.platform
if (hostos == 'mac' or hostos == 'darwin'):
localAutomation.IS_MAC = True
elif (hostos == 'linux' or hostos == 'linux2'):
localAutomation.IS_LINUX = True
localAutomation.UNIXISH = True
elif (hostos == 'win32' or hostos == 'win64'):
localAutomation.BIN_SUFFIX = ".exe"
localAutomation.IS_WIN32 = True
return localAutomation
# This seems kludgy, but this class uses paths from the remote host in the
# options, except when calling up to the base class, which doesn't
# understand the distinction. This switches out the remote values for local
# ones that the base class understands. This is necessary for the web
# server, SSL tunnel and profile building functions.
def switchToLocalPaths(self, options):
""" Set local paths in the options, return a function that will restore remote values """
remoteXrePath = options.xrePath
remoteProfilePath = options.profilePath
remoteUtilityPath = options.utilityPath
localAutomation = self.makeLocalAutomation()
paths = [
options.xrePath,
localAutomation.DIST_BIN,
]
options.xrePath = self.findPath(paths)
if options.xrePath is None:
self.log.error(
"unable to find xulrunner path for %s, please specify with --xre-path"
% os.name)
sys.exit(1)
xpcshell = "xpcshell"
if (os.name == "nt"):
xpcshell += ".exe"
if options.utilityPath:
paths = [options.utilityPath, options.xrePath]
else:
paths = [options.xrePath]
options.utilityPath = self.findPath(paths, xpcshell)
if options.utilityPath is None:
self.log.error(
"unable to find utility path for %s, please specify with --utility-path"
% os.name)
sys.exit(1)
xpcshell_path = os.path.join(options.utilityPath, xpcshell)
if localAutomation.elf_arm(xpcshell_path):
self.log.error('xpcshell at %s is an ARM binary; please use '
'the --utility-path argument to specify the path '
'to a desktop version.' % xpcshell_path)
sys.exit(1)
if self.localProfile:
options.profilePath = self.localProfile
else:
options.profilePath = None
def fixup():
options.xrePath = remoteXrePath
options.utilityPath = remoteUtilityPath
options.profilePath = remoteProfilePath
return fixup
def startServers(self, options, debuggerInfo):
""" Create the servers on the host and start them up """
restoreRemotePaths = self.switchToLocalPaths(options)
# ignoreSSLTunnelExts is a workaround for bug 1109310
MochitestDesktop.startServers(self,
options,
debuggerInfo,
ignoreSSLTunnelExts=True)
restoreRemotePaths()
def buildProfile(self, options):
restoreRemotePaths = self.switchToLocalPaths(options)
if options.testingModulesDir:
try:
self.device.push(options.testingModulesDir,
self.remoteModulesDir)
self.device.chmod(self.remoteModulesDir, recursive=True)
except Exception:
self.log.error(
"Automation Error: Unable to copy test modules to device.")
raise
savedTestingModulesDir = options.testingModulesDir
options.testingModulesDir = self.remoteModulesDir
else:
savedTestingModulesDir = None
manifest = MochitestDesktop.buildProfile(self, options)
if savedTestingModulesDir:
options.testingModulesDir = savedTestingModulesDir
self.localProfile = options.profilePath
restoreRemotePaths()
options.profilePath = self.remoteProfile
return manifest
def buildURLOptions(self, options, env):
saveLogFile = options.logFile
options.logFile = self.remoteLogFile
options.profilePath = self.localProfile
env["MOZ_HIDE_RESULTS_TABLE"] = "1"
retVal = MochitestDesktop.buildURLOptions(self, options, env)
# we really need testConfig.js (for browser chrome)
try:
self.device.push(options.profilePath, self.remoteProfile)
self.device.chmod(self.remoteProfile, recursive=True)
except Exception:
self.log.error(
"Automation Error: Unable to copy profile to device.")
raise
options.profilePath = self.remoteProfile
options.logFile = saveLogFile
return retVal
def getChromeTestDir(self, options):
local = super(MochiRemote, self).getChromeTestDir(options)
remote = self.remoteChromeTestDir
if options.flavor == 'chrome' and not self.chromePushed:
self.log.info("pushing %s to %s on device..." % (local, remote))
local = os.path.join(local, "chrome")
self.device.push(local, remote)
self.chromePushed = True
return remote
def getLogFilePath(self, logFile):
return logFile
def printDeviceInfo(self, printLogcat=False):
try:
if printLogcat:
logcat = self.device.get_logcat(
filter_out_regexps=fennecLogcatFilters)
self.log.info('\n' +
''.join(logcat).decode('utf-8', 'replace'))
self.log.info("Device info:")
devinfo = self.device.get_info()
for category in devinfo:
if type(devinfo[category]) is list:
self.log.info(" %s:" % category)
for item in devinfo[category]:
self.log.info(" %s" % item)
else:
self.log.info(" %s: %s" % (category, devinfo[category]))
self.log.info("Test root: %s" % self.device.test_root)
except Exception as e:
self.log.warning("Error getting device information: %s" % str(e))
def getGMPPluginPath(self, options):
# TODO: bug 1149374
return None
def buildBrowserEnv(self, options, debugger=False):
browserEnv = MochitestDesktop.buildBrowserEnv(self,
options,
debugger=debugger)
# remove desktop environment not used on device
if "XPCOM_MEM_BLOAT_LOG" in browserEnv:
del browserEnv["XPCOM_MEM_BLOAT_LOG"]
# override mozLogs to avoid processing in MochitestDesktop base class
self.mozLogs = None
browserEnv["MOZ_LOG_FILE"] = os.path.join(self.remoteMozLog,
self.mozLogName)
if options.dmd:
browserEnv['DMD'] = '1'
return browserEnv
def runApp(self, *args, **kwargs):
"""front-end automation.py's `runApp` functionality until FennecRunner is written"""
# automation.py/remoteautomation `runApp` takes the profile path,
# whereas runtest.py's `runApp` takes a mozprofile object.
if 'profileDir' not in kwargs and 'profile' in kwargs:
kwargs['profileDir'] = kwargs.pop('profile').profile
# remove args not supported by automation.py
kwargs.pop('marionette_args', None)
ret, _ = self.automation.runApp(*args, **kwargs)
self.countpass += self.counts['pass']
self.countfail += self.counts['fail']
self.counttodo += self.counts['todo']
return ret, None
class RemoteCPPUnitTests(cppunittests.CPPUnitTests):
def __init__(self, options, progs):
cppunittests.CPPUnitTests.__init__(self)
self.options = options
self.device = ADBAndroid(adb=options.adb_path or 'adb',
device=options.device_serial,
test_root=options.remote_test_root)
self.remote_test_root = posixpath.join(self.device.test_root, "cppunittests")
self.remote_bin_dir = posixpath.join(self.remote_test_root, "b")
self.remote_tmp_dir = posixpath.join(self.remote_test_root, "tmp")
self.remote_home_dir = posixpath.join(self.remote_test_root, "h")
if options.setup:
self.setup_bin(progs)
def setup_bin(self, progs):
self.device.rm(self.remote_test_root, force=True, recursive=True)
self.device.mkdir(self.remote_home_dir, parents=True)
self.device.mkdir(self.remote_tmp_dir)
self.push_libs()
self.push_progs(progs)
self.device.chmod(self.remote_bin_dir, recursive=True, root=True)
def push_libs(self):
if self.options.local_apk:
with mozfile.TemporaryDirectory() as tmpdir:
apk_contents = ZipFile(self.options.local_apk)
for info in apk_contents.infolist():
if info.filename.endswith(".so"):
print >> sys.stderr, "Pushing %s.." % info.filename
remote_file = posixpath.join(
self.remote_bin_dir, os.path.basename(info.filename))
apk_contents.extract(info, tmpdir)
local_file = os.path.join(tmpdir, info.filename)
with open(local_file) as f:
# Decompress xz-compressed file.
if f.read(5)[1:] == '7zXZ':
cmd = [
'xz', '-df', '--suffix', '.so', local_file]
subprocess.check_output(cmd)
# xz strips the ".so" file suffix.
os.rename(local_file[:-3], local_file)
self.device.push(local_file, remote_file)
elif self.options.local_lib:
for file in os.listdir(self.options.local_lib):
if file.endswith(".so"):
print >> sys.stderr, "Pushing %s.." % file
remote_file = posixpath.join(self.remote_bin_dir, file)
local_file = os.path.join(self.options.local_lib, file)
self.device.push(local_file, remote_file)
# Additional libraries may be found in a sub-directory such as
# "lib/armeabi-v7a"
for subdir in ["assets", "lib"]:
local_arm_lib = os.path.join(self.options.local_lib, subdir)
if os.path.isdir(local_arm_lib):
for root, dirs, files in os.walk(local_arm_lib):
for file in files:
if (file.endswith(".so")):
print >> sys.stderr, "Pushing %s.." % file
remote_file = posixpath.join(
self.remote_bin_dir, file)
local_file = os.path.join(root, file)
self.device.push(local_file, remote_file)
def push_progs(self, progs):
for local_file in progs:
remote_file = posixpath.join(
self.remote_bin_dir, os.path.basename(local_file))
self.device.push(local_file, remote_file)
def build_environment(self):
env = self.build_core_environment()
env['LD_LIBRARY_PATH'] = self.remote_bin_dir
env["TMPDIR"] = self.remote_tmp_dir
env["HOME"] = self.remote_home_dir
env["MOZ_XRE_DIR"] = self.remote_bin_dir
if self.options.add_env:
for envdef in self.options.add_env:
envdef_parts = envdef.split("=", 1)
if len(envdef_parts) == 2:
env[envdef_parts[0]] = envdef_parts[1]
elif len(envdef_parts) == 1:
env[envdef_parts[0]] = ""
else:
self.log.warning(
"invalid --addEnv option skipped: %s" % envdef)
return env
def run_one_test(self, prog, env, symbols_path=None, interactive=False,
timeout_factor=1):
"""
Run a single C++ unit test program remotely.
Arguments:
* prog: The path to the test program to run.
* env: The environment to use for running the program.
* symbols_path: A path to a directory containing Breakpad-formatted
symbol files for producing stack traces on crash.
* timeout_factor: An optional test-specific timeout multiplier.
Return True if the program exits with a zero status, False otherwise.
"""
basename = os.path.basename(prog)
remote_bin = posixpath.join(self.remote_bin_dir, basename)
self.log.test_start(basename)
test_timeout = cppunittests.CPPUnitTests.TEST_PROC_TIMEOUT * \
timeout_factor
try:
output = self.device.shell_output(remote_bin, env=env,
cwd=self.remote_home_dir,
timeout=test_timeout)
returncode = 0
except ADBProcessError as e:
output = e.adb_process.stdout
returncode = e.adb_process.exitcode
self.log.process_output(basename, "\n%s" % output,
command=[remote_bin])
with mozfile.TemporaryDirectory() as tempdir:
self.device.pull(self.remote_home_dir, tempdir)
if mozcrash.check_for_crashes(tempdir, symbols_path,
test_name=basename):
self.log.test_end(basename, status='CRASH', expected='PASS')
return False
result = returncode == 0
if not result:
self.log.test_end(basename, status='FAIL', expected='PASS',
message=("test failed with return code %s" %
returncode))
else:
self.log.test_end(basename, status='PASS', expected='PASS')
return result
def cli(record, certutil, url, path):
# create profile
profile = create_profile("firefox")
print("Created profile: {}".format(profile.profile))
mitmproxy_home = os.path.join(os.path.expanduser("~"), ".mitmproxy")
cert = os.path.join(mitmproxy_home, "mitmproxy-ca-cert.cer")
# start mitmdump
scripts = os.path.join(os.getcwd(), "scripts")
mitmdump = os.path.join(os.getcwd(), "utils", "mitmdump")
if record:
command = [mitmdump, "--wfile", path]
else:
command = [
mitmdump,
"--replay-kill-extra",
"--script",
" ".join(
[os.path.join(scripts, "alternate-server-replay.py"), path]),
]
try:
print(command)
mitmdump_process = subprocess.Popen(command)
# create certificate database
certdb = "sql:{}/".format(profile.profile)
print("Creating certificate database")
command = [certutil, "-N", "-v", "-d", certdb, "--empty-password"]
subprocess.call(command)
# install mitmproxy certificate
command = [
certutil,
"-A",
"-d",
certdb,
"-n",
"mitmproxy-cert",
"-t",
"TC,,",
"-a",
"-i",
cert,
]
print("Installing {} into certificate database".format(cert))
subprocess.call(command)
# verify certificate is installed
command = [certutil, "-d", certdb, "-L"]
assert "mitmproxy-cert" in subprocess.check_output(command)
# setup device
device = ADBAndroid()
device.create_socket_connection("reverse", "tcp:8080", "tcp:8080")
device_storage = "/sdcard/raptor"
device_profile = os.path.join(device_storage, "profile")
if device.is_dir(device_storage):
device.rm(device_storage, recursive=True)
device.mkdir(device_storage)
device.mkdir(device_profile)
userjs = os.path.join(profile.profile, "user.js")
with open(userjs) as f:
prefs = f.readlines()
prefs = [p for p in prefs if "network.proxy" not in p]
with open(userjs, "w") as f:
f.writelines(prefs)
proxy = {
"network.proxy.type": 1,
"network.proxy.http": "127.0.0.1",
"network.proxy.http_port": 8080,
"network.proxy.ssl": "127.0.0.1",
"network.proxy.ssl_port": 8080,
"network.proxy.no_proxies_on": "localhost, 127.0.0.1",
}
profile.set_preferences(proxy)
device.push(profile.profile, device_profile)
device.chmod(device_storage, recursive=True)
app_args = [
"-profile",
device_profile,
"--marionette",
"--es",
"env0",
"LOG_VERBOSE=1",
"--es",
"env1",
"R_LOG_LEVEL=6",
]
# start app
app_name = "org.mozilla.geckoview_example"
activity_name = "GeckoViewActivity"
device.stop_application(app_name)
device.launch_activity(
app_name,
activity_name,
extra_args=app_args,
url=url,
e10s=True,
fail_if_running=False,
)
# wait for mitmdump to finish
mitmdump_process.wait()
finally:
if mitmdump_process is None:
mitmdump_process.terminate()
exit(mitmdump_process.returncode)
class XPCShellRemote(xpcshell.XPCShellTests, object):
def __init__(self, options, log):
xpcshell.XPCShellTests.__init__(self, log)
self.options = options
verbose = False
if options['log_tbpl_level'] == 'debug' or options[
'log_mach_level'] == 'debug':
verbose = True
self.device = ADBAndroid(adb=options['adbPath'],
device=options['deviceSerial'],
test_root=options['remoteTestRoot'],
verbose=verbose)
self.remoteTestRoot = posixpath.join(self.device.test_root, "xpc")
# Add Android version (SDK level) to mozinfo so that manifest entries
# can be conditional on android_version.
mozinfo.info['android_version'] = self.device.version
self.localLib = options['localLib']
self.localBin = options['localBin']
self.pathMapping = []
# remoteBinDir contains xpcshell and its wrapper script, both of which must
# be executable. Since +x permissions cannot usually be set on /mnt/sdcard,
# and the test root may be on /mnt/sdcard, remoteBinDir is set to be on
# /data/local, always.
self.remoteBinDir = posixpath.join("/data", "local", "xpcb")
# Terse directory names are used here ("c" for the components directory)
# to minimize the length of the command line used to execute
# xpcshell on the remote device. adb has a limit to the number
# of characters used in a shell command, and the xpcshell command
# line can be quite complex.
self.remoteTmpDir = posixpath.join(self.remoteTestRoot, "tmp")
self.remoteScriptsDir = self.remoteTestRoot
self.remoteComponentsDir = posixpath.join(self.remoteTestRoot, "c")
self.remoteModulesDir = posixpath.join(self.remoteTestRoot, "m")
self.remoteMinidumpDir = posixpath.join(self.remoteTestRoot,
"minidumps")
self.remoteClearDirScript = posixpath.join(self.remoteBinDir,
"cleardir")
self.profileDir = posixpath.join(self.remoteTestRoot, "p")
self.remoteDebugger = options['debugger']
self.remoteDebuggerArgs = options['debuggerArgs']
self.testingModulesDir = options['testingModulesDir']
self.env = {}
if options['objdir']:
self.xpcDir = os.path.join(options['objdir'], "_tests/xpcshell")
elif os.path.isdir(os.path.join(here, 'tests')):
self.xpcDir = os.path.join(here, 'tests')
else:
print("Couldn't find local xpcshell test directory",
file=sys.stderr)
sys.exit(1)
if options['localAPK']:
self.localAPKContents = ZipFile(options['localAPK'])
if options['setup']:
self.setupTestDir()
self.setupUtilities()
self.setupModules()
self.setupMinidumpDir()
self.remoteAPK = None
if options['localAPK']:
self.remoteAPK = posixpath.join(
self.remoteBinDir, os.path.basename(options['localAPK']))
self.setAppRoot()
# data that needs to be passed to the RemoteXPCShellTestThread
self.mobileArgs = {
'device': self.device,
'remoteBinDir': self.remoteBinDir,
'remoteScriptsDir': self.remoteScriptsDir,
'remoteComponentsDir': self.remoteComponentsDir,
'remoteModulesDir': self.remoteModulesDir,
'options': self.options,
'remoteDebugger': self.remoteDebugger,
'pathMapping': self.pathMapping,
'profileDir': self.profileDir,
'remoteTmpDir': self.remoteTmpDir,
'remoteMinidumpDir': self.remoteMinidumpDir,
'remoteClearDirScript': self.remoteClearDirScript,
}
if self.remoteAPK:
self.mobileArgs['remoteAPK'] = self.remoteAPK
def setLD_LIBRARY_PATH(self):
self.env["LD_LIBRARY_PATH"] = self.remoteBinDir
def pushWrapper(self):
# Rather than executing xpcshell directly, this wrapper script is
# used. By setting environment variables and the cwd in the script,
# the length of the per-test command line is shortened. This is
# often important when using ADB, as there is a limit to the length
# of the ADB command line.
localWrapper = tempfile.mktemp()
f = open(localWrapper, "w")
f.write("#!/system/bin/sh\n")
for envkey, envval in self.env.iteritems():
f.write("export %s=%s\n" % (envkey, envval))
f.writelines([
"cd $1\n", "echo xpcw: cd $1\n", "shift\n",
"echo xpcw: xpcshell \"$@\"\n",
"%s/xpcshell \"$@\"\n" % self.remoteBinDir
])
f.close()
remoteWrapper = posixpath.join(self.remoteBinDir, "xpcw")
self.device.push(localWrapper, remoteWrapper)
os.remove(localWrapper)
# Removing and re-creating a directory is a common operation which
# can be implemented more efficiently with a shell script.
localWrapper = tempfile.mktemp()
f = open(localWrapper, "w")
# The directory may not exist initially, so rm may fail. 'rm -f' is not
# supported on some Androids. Similarly, 'test' and 'if [ -d ]' are not
# universally available, so we just ignore errors from rm.
f.writelines(
["#!/system/bin/sh\n", "rm -r \"$1\"\n", "mkdir \"$1\"\n"])
f.close()
self.device.push(localWrapper, self.remoteClearDirScript)
os.remove(localWrapper)
self.device.chmod(self.remoteBinDir, recursive=True)
def buildEnvironment(self):
self.buildCoreEnvironment()
self.setLD_LIBRARY_PATH()
self.env["MOZ_LINKER_CACHE"] = self.remoteBinDir
if self.options['localAPK'] and self.appRoot:
self.env["GRE_HOME"] = self.appRoot
self.env["XPCSHELL_TEST_PROFILE_DIR"] = self.profileDir
self.env["TMPDIR"] = self.remoteTmpDir
self.env["HOME"] = self.profileDir
self.env["XPCSHELL_TEST_TEMP_DIR"] = self.remoteTmpDir
self.env["XPCSHELL_MINIDUMP_DIR"] = self.remoteMinidumpDir
if self.options['setup']:
self.pushWrapper()
def setAppRoot(self):
# Determine the application root directory associated with the package
# name used by the Fennec APK.
self.appRoot = None
packageName = None
if self.options['localAPK']:
try:
packageName = self.localAPKContents.read("package-name.txt")
if packageName:
self.appRoot = posixpath.join("/data", "data",
packageName.strip())
except Exception as detail:
print("unable to determine app root: " + str(detail))
pass
return None
def setupUtilities(self):
self.device.rm(self.remoteTmpDir, force=True, recursive=True)
self.device.mkdir(self.remoteTmpDir)
self.device.rm(self.remoteBinDir, force=True, recursive=True)
remotePrefDir = posixpath.join(self.remoteBinDir, "defaults", "pref")
self.device.mkdir(posixpath.join(remotePrefDir, "extra"), parents=True)
self.device.mkdir(self.remoteScriptsDir, parents=True)
self.device.mkdir(self.remoteComponentsDir, parents=True)
local = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'head.js')
remoteFile = posixpath.join(self.remoteScriptsDir, "head.js")
self.device.push(local, remoteFile)
# The xpcshell binary is required for all tests. Additional binaries
# are required for some tests. This list should be similar to
# TEST_HARNESS_BINS in testing/mochitest/Makefile.in.
binaries = [
"xpcshell", "ssltunnel", "certutil", "pk12util", "BadCertServer",
"OCSPStaplingServer", "GenerateOCSPResponse",
"SymantecSanctionsServer"
]
for fname in binaries:
local = os.path.join(self.localBin, fname)
if os.path.isfile(local):
print("Pushing %s.." % fname, file=sys.stderr)
remoteFile = posixpath.join(self.remoteBinDir, fname)
self.device.push(local, remoteFile)
else:
print("*** Expected binary %s not found in %s!" %
(fname, self.localBin),
file=sys.stderr)
local = os.path.join(self.localBin, "components/httpd.js")
remoteFile = posixpath.join(self.remoteComponentsDir, "httpd.js")
self.device.push(local, remoteFile)
local = os.path.join(self.localBin, "components/httpd.manifest")
remoteFile = posixpath.join(self.remoteComponentsDir, "httpd.manifest")
self.device.push(local, remoteFile)
if self.options['localAPK']:
remoteFile = posixpath.join(
self.remoteBinDir, os.path.basename(self.options['localAPK']))
self.device.push(self.options['localAPK'], remoteFile)
self.pushLibs()
def pushLibs(self):
pushed_libs_count = 0
if self.options['localAPK']:
try:
dir = tempfile.mkdtemp()
for info in self.localAPKContents.infolist():
if info.filename.endswith(".so"):
print("Pushing %s.." % info.filename, file=sys.stderr)
remoteFile = posixpath.join(
self.remoteBinDir, os.path.basename(info.filename))
self.localAPKContents.extract(info, dir)
localFile = os.path.join(dir, info.filename)
with open(localFile) as f:
# Decompress xz-compressed file.
if f.read(5)[1:] == '7zXZ':
cmd = [
'xz', '-df', '--suffix', '.so', localFile
]
subprocess.check_output(cmd)
# xz strips the ".so" file suffix.
os.rename(localFile[:-3], localFile)
self.device.push(localFile, remoteFile)
pushed_libs_count += 1
finally:
shutil.rmtree(dir)
return pushed_libs_count
for file in os.listdir(self.localLib):
if (file.endswith(".so")):
print("Pushing %s.." % file, file=sys.stderr)
if 'libxul' in file:
print("This is a big file, it could take a while.",
file=sys.stderr)
localFile = os.path.join(self.localLib, file)
remoteFile = posixpath.join(self.remoteBinDir, file)
self.device.push(localFile, remoteFile)
pushed_libs_count += 1
# Additional libraries may be found in a sub-directory such as "lib/armeabi-v7a"
localArmLib = os.path.join(self.localLib, "lib")
if os.path.exists(localArmLib):
for root, dirs, files in os.walk(localArmLib):
for file in files:
if (file.endswith(".so")):
print("Pushing %s.." % file, file=sys.stderr)
localFile = os.path.join(root, file)
remoteFile = posixpath.join(self.remoteBinDir, file)
self.device.push(localFile, remoteFile)
pushed_libs_count += 1
return pushed_libs_count
def setupModules(self):
if self.testingModulesDir:
self.device.push(self.testingModulesDir, self.remoteModulesDir)
def setupTestDir(self):
print('pushing %s' % self.xpcDir)
# The tests directory can be quite large: 5000 files and growing!
# Sometimes - like on a low-end aws instance running an emulator - the push
# may exceed the default 5 minute timeout, so we increase it here to 10 minutes.
self.device.rm(self.remoteTestRoot, force=True, recursive=True)
self.device.push(self.xpcDir, self.remoteScriptsDir, timeout=600)
def setupMinidumpDir(self):
self.device.rm(self.remoteMinidumpDir, force=True, recursive=True)
self.device.mkdir(self.remoteMinidumpDir)
def buildTestList(self, test_tags=None, test_paths=None, verify=False):
xpcshell.XPCShellTests.buildTestList(self,
test_tags=test_tags,
test_paths=test_paths,
verify=verify)
uniqueTestPaths = set([])
for test in self.alltests:
uniqueTestPaths.add(test['here'])
for testdir in uniqueTestPaths:
abbrevTestDir = os.path.relpath(testdir, self.xpcDir)
remoteScriptDir = posixpath.join(self.remoteScriptsDir,
abbrevTestDir)
self.pathMapping.append(PathMapping(testdir, remoteScriptDir))
class Raptor(object):
"""Container class for Raptor"""
def __init__(self,
app,
binary,
run_local=False,
obj_path=None,
gecko_profile=False,
gecko_profile_interval=None,
gecko_profile_entries=None,
symbols_path=None,
host=None,
is_release_build=False,
debug_mode=False):
self.config = {}
self.config['app'] = app
self.config['binary'] = binary
self.config['platform'] = mozinfo.os
self.config['processor'] = mozinfo.processor
self.config['run_local'] = run_local
self.config['obj_path'] = obj_path
self.config['gecko_profile'] = gecko_profile
self.config['gecko_profile_interval'] = gecko_profile_interval
self.config['gecko_profile_entries'] = gecko_profile_entries
self.config['symbols_path'] = symbols_path
self.config['host'] = host
self.config['is_release_build'] = is_release_build
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
self.log = get_default_logger(component='raptor-main')
self.control_server = None
self.playback = None
self.benchmark = None
self.gecko_profiler = None
self.post_startup_delay = 30000
self.device = None
# debug mode is currently only supported when running locally
self.debug_mode = debug_mode if self.config['run_local'] else False
# if running debug-mode reduce the pause after browser startup
if self.debug_mode:
self.post_startup_delay = 3000
self.log.info(
"debug-mode enabled, reducing post-browser startup pause to %d ms"
% self.post_startup_delay)
# Create the profile; for geckoview we want a firefox profile type
if self.config['app'] == 'geckoview':
self.profile = create_profile('firefox')
else:
self.profile = create_profile(self.config['app'])
# Merge in base profiles
with open(os.path.join(self.profile_data_dir, 'profiles.json'),
'r') as fh:
base_profiles = json.load(fh)['raptor']
for name in base_profiles:
path = os.path.join(self.profile_data_dir, name)
self.log.info("Merging profile: {}".format(path))
self.profile.merge(path)
# add profile dir to our config
self.config['local_profile_dir'] = self.profile.profile
# create results holder
self.results_handler = RaptorResultsHandler()
# when testing desktop browsers we use mozrunner to start the browser; when
# testing on android (i.e. geckoview) we use mozdevice to control the device app
if self.config['app'] == "geckoview":
# create the android device handler; it gets initiated and sets up adb etc
self.log.info("creating android device handler using mozdevice")
self.device = ADBAndroid(verbose=True)
self.device.clear_logcat()
else:
# create the desktop browser runner
self.log.info("creating browser runner using mozrunner")
self.output_handler = OutputHandler()
process_args = {
'processOutputLine': [self.output_handler],
}
runner_cls = runners[app]
self.runner = runner_cls(binary,
profile=self.profile,
process_args=process_args)
self.log.info("raptor config: %s" % str(self.config))
@property
def profile_data_dir(self):
if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
'testing', 'profiles')
if build:
return os.path.join(build.topsrcdir, 'testing', 'profiles')
return os.path.join(here, 'profile_data')
def start_control_server(self):
self.control_server = RaptorControlServer(self.results_handler,
self.debug_mode)
self.control_server.start()
# for android we must make the control server available to the device
if self.config['app'] == "geckoview" and self.config['host'] in (
'localhost', '127.0.0.1'):
self.log.info(
"making the raptor control server port available to device")
_tcp_port = "tcp:%s" % self.control_server.port
self.device.create_socket_connection('reverse', _tcp_port,
_tcp_port)
def get_playback_config(self, test):
self.config['playback_tool'] = test.get('playback')
self.log.info("test uses playback tool: %s " %
self.config['playback_tool'])
self.config['playback_binary_manifest'] = test.get(
'playback_binary_manifest', None)
_key = 'playback_binary_zip_%s' % self.config['platform']
self.config['playback_binary_zip'] = test.get(_key, None)
self.config['playback_pageset_manifest'] = test.get(
'playback_pageset_manifest', None)
_key = 'playback_pageset_zip_%s' % self.config['platform']
self.config['playback_pageset_zip'] = test.get(_key, None)
self.config['playback_recordings'] = test.get('playback_recordings',
None)
self.config['python3_win_manifest'] = test.get('python3_win_manifest',
None)
def run_test(self, test, timeout=None):
self.log.info("starting raptor test: %s" % test['name'])
self.log.info("test settings: %s" % str(test))
self.log.info("raptor config: %s" % str(self.config))
# benchmark-type tests require the benchmark test to be served out
if test.get('type') == "benchmark":
self.benchmark = Benchmark(self.config, test)
benchmark_port = int(self.benchmark.port)
# for android we must make the benchmarks server available to the device
if self.config['app'] == "geckoview" and self.config['host'] \
in ('localhost', '127.0.0.1'):
self.log.info(
"making the raptor benchmarks server port available to device"
)
_tcp_port = "tcp:%s" % benchmark_port
self.device.create_socket_connection('reverse', _tcp_port,
_tcp_port)
else:
benchmark_port = 0
gen_test_config(self.config['app'],
test['name'],
self.control_server.port,
self.post_startup_delay,
host=self.config['host'],
b_port=benchmark_port,
debug_mode=1 if self.debug_mode else 0)
# must intall raptor addon each time because we dynamically update some content
# note: for chrome the addon is just a list of paths that ultimately are added
# to the chromium command line '--load-extension' argument
raptor_webext = os.path.join(webext_dir, 'raptor')
self.log.info("installing webext %s" % raptor_webext)
self.profile.addons.install(raptor_webext)
# add test specific preferences
if test.get("preferences", None) is not None:
if self.config['app'] == "firefox":
self.profile.set_preferences(json.loads(test['preferences']))
else:
self.log.info("preferences were configured for the test, \
but we do not install them on non Firefox browsers."
)
# on firefox we can get an addon id; chrome addon actually is just cmd line arg
if self.config['app'] in ["firefox", "geckoview"]:
webext_id = self.profile.addons.addon_details(raptor_webext)['id']
# for android/geckoview, create a top-level raptor folder on the device
# sdcard; if it already exists remove it so we start fresh each time
if self.config['app'] == "geckoview":
self.device_raptor_dir = "/sdcard/raptor"
self.config['device_raptor_dir'] = self.device_raptor_dir
if self.device.is_dir(self.device_raptor_dir):
self.log.info("deleting existing device raptor dir: %s" %
self.device_raptor_dir)
self.device.rm(self.device_raptor_dir, recursive=True)
self.log.info("creating raptor folder on sdcard: %s" %
self.device_raptor_dir)
self.device.mkdir(self.device_raptor_dir)
self.device.chmod(self.device_raptor_dir, recursive=True)
# some tests require tools to playback the test pages
if test.get('playback', None) is not None:
self.get_playback_config(test)
# startup the playback tool
self.playback = get_playback(self.config, self.device)
# for android we must make the playback server available to the device
if self.config['app'] == "geckoview" and self.config['host'] \
in ('localhost', '127.0.0.1'):
self.log.info(
"making the raptor playback server port available to device"
)
_tcp_port = "tcp:8080"
self.device.create_socket_connection('reverse', _tcp_port,
_tcp_port)
if self.config['app'] in ("geckoview", "firefox") and \
self.config['host'] not in ('localhost', '127.0.0.1'):
# Must delete the proxy settings from the profile if running
# the test with a host different from localhost.
userjspath = os.path.join(self.profile.profile, 'user.js')
with open(userjspath) as userjsfile:
prefs = userjsfile.readlines()
prefs = [pref for pref in prefs if 'network.proxy' not in pref]
with open(userjspath, 'w') as userjsfile:
userjsfile.writelines(prefs)
# for geckoview/android pageload playback we can't use a policy to turn on the
# proxy; we need to set prefs instead; note that the 'host' may be different
# than '127.0.0.1' so we must set the prefs accordingly
if self.config['app'] == "geckoview" and test.get('playback',
None) is not None:
self.log.info(
"setting profile prefs to turn on the geckoview browser proxy")
no_proxies_on = "localhost, 127.0.0.1, %s" % self.config['host']
proxy_prefs = {}
proxy_prefs["network.proxy.type"] = 1
proxy_prefs["network.proxy.http"] = self.config['host']
proxy_prefs["network.proxy.http_port"] = 8080
proxy_prefs["network.proxy.ssl"] = self.config['host']
proxy_prefs["network.proxy.ssl_port"] = 8080
proxy_prefs["network.proxy.no_proxies_on"] = no_proxies_on
self.profile.set_preferences(proxy_prefs)
# now some final settings, and then startup of the browser under test
if self.config['app'] == "geckoview":
# for android/geckoview we must copy the profile onto the device and set perms
if not self.device.is_app_installed(self.config['binary']):
raise Exception('%s is not installed' % self.config['binary'])
self.device_profile = os.path.join(self.device_raptor_dir,
"profile")
if self.device.is_dir(self.device_profile):
self.log.info("deleting existing device profile folder: %s" %
self.device_profile)
self.device.rm(self.device_profile, recursive=True)
self.log.info("creating profile folder on device: %s" %
self.device_profile)
self.device.mkdir(self.device_profile)
self.log.info("copying firefox profile onto the device")
self.log.info("note: the profile folder being copied is: %s" %
self.profile.profile)
self.log.info(
'the adb push cmd copies that profile dir to a new temp dir before copy'
)
self.device.push(self.profile.profile, self.device_profile)
self.device.chmod(self.device_profile, recursive=True)
# now start the geckoview app
self.log.info("starting %s" % self.config['app'])
extra_args = [
"-profile", self.device_profile, "--es", "env0",
"LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6"
]
try:
# make sure the geckoview app is not running before
# attempting to start.
self.device.stop_application(self.config['binary'])
self.device.launch_activity(self.config['binary'],
"GeckoViewActivity",
extra_args=extra_args,
url='about:blank',
e10s=True,
fail_if_running=False)
except Exception:
self.log.error("Exception launching %s" %
self.config['binary'])
raise
self.control_server.device = self.device
self.control_server.app_name = self.config['binary']
else:
# For Firefox we need to set
# MOZ_DISABLE_NONLOCAL_CONNECTIONS=1 env var before
# startup when testing release builds from mozilla-beta or
# mozilla-release. This is because of restrictions on
# release builds that require webextensions to be signed
# unless MOZ_DISABLE_NONLOCAL_CONNECTIONS is set to '1'.
if self.config['app'] == "firefox" and self.config[
'is_release_build']:
self.log.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=1")
os.environ['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = "1"
# if running debug-mode, tell Firefox to open the browser console on startup
# for google chrome, open the devtools on the raptor test tab
if self.debug_mode:
if self.config['app'] == "firefox":
self.runner.cmdargs.extend(['-jsconsole'])
if self.config['app'] == "chrome":
self.runner.cmdargs.extend(
['--auto-open-devtools-for-tabs'])
# now start the desktop browser
self.log.info("starting %s" % self.config['app'])
# if running a pageload test on google chrome, add the cmd line options
# to turn on the proxy and ignore security certificate errors
# if using host localhost, 127.0.0.1.
if self.config['app'] == "chrome" and test.get('playback',
None) is not None:
chrome_args = [
'--proxy-server="http=127.0.0.1:8080;' +
'https=127.0.0.1:8080;ssl=127.0.0.1:8080"',
'--ignore-certificate-errors'
]
if self.config['host'] not in ('localhost', '127.0.0.1'):
chrome_args[0] = chrome_args[0].replace(
'127.0.0.1', self.config['host'])
if ' '.join(chrome_args) not in ' '.join(self.runner.cmdargs):
self.runner.cmdargs.extend(chrome_args)
self.runner.start()
proc = self.runner.process_handler
self.output_handler.proc = proc
self.control_server.browser_proc = proc
# pageload tests need to be able to access non-local connections via mitmproxy
if self.config['app'] == "firefox" and self.config['is_release_build'] and \
test.get('playback', None) is not None:
self.log.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=0")
os.environ['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = "0"
# if geckoProfile is enabled, initialize it
if self.config['gecko_profile'] is True:
self._init_gecko_profiling(test)
# tell the control server the gecko_profile dir; the control server will
# receive the actual gecko profiles from the web ext and will write them
# to disk; then profiles are picked up by gecko_profile.symbolicate
self.control_server.gecko_profile_dir = self.gecko_profiler.gecko_profile_dir
# set our cs flag to indicate we are running the browser/app
self.control_server._finished = False
# convert to seconds and account for page cycles
timeout = int(timeout / 1000) * int(test['page_cycles'])
# account for the pause the raptor webext runner takes after browser startup
timeout += int(self.post_startup_delay / 1000)
# if geckoProfile enabled, give browser more time for profiling
if self.config['gecko_profile'] is True:
timeout += 5 * 60
try:
elapsed_time = 0
while not self.control_server._finished:
time.sleep(1)
# we only want to force browser-shutdown on timeout if not in debug mode;
# in debug-mode we leave the browser running (require manual shutdown)
if not self.debug_mode:
elapsed_time += 1
if elapsed_time > (timeout) - 5: # stop 5 seconds early
self.log.info(
"application timed out after {} seconds".format(
timeout))
self.control_server.wait_for_quit()
break
finally:
if self.config['app'] != "geckoview":
try:
self.runner.check_for_crashes()
except NotImplementedError: # not implemented for Chrome
pass
# TODO: if on geckoview is there some cleanup here i.e. check for crashes?
if self.playback is not None:
self.playback.stop()
# remove the raptor webext; as it must be reloaded with each subtest anyway
self.log.info("removing webext %s" % raptor_webext)
if self.config['app'] in ["firefox", "geckoview"]:
self.profile.addons.remove_addon(webext_id)
# for chrome the addon is just a list (appended to cmd line)
if self.config['app'] in ["chrome", "chrome-android"]:
self.profile.addons.remove(raptor_webext)
# gecko profiling symbolication
if self.config['gecko_profile'] is True:
self.gecko_profiler.symbolicate()
# clean up the temp gecko profiling folders
self.log.info("cleaning up after gecko profiling")
self.gecko_profiler.clean()
# browser should be closed by now but this is a backup-shutdown (if not in debug-mode)
if not self.debug_mode:
if self.config['app'] != "geckoview":
if self.runner.is_running():
self.runner.stop()
# TODO the geckoview app should have been shutdown by this point by the
# control server, but we can double-check here to make sure
else:
# in debug mode, and running locally, leave the browser running
if self.config['run_local']:
self.log.info(
"* debug-mode enabled - please shutdown the browser manually..."
)
self.runner.wait(timeout=None)
def _init_gecko_profiling(self, test):
self.log.info("initializing gecko profiler")
upload_dir = os.getenv('MOZ_UPLOAD_DIR')
if not upload_dir:
self.log.critical(
"Profiling ignored because MOZ_UPLOAD_DIR was not set")
else:
self.gecko_profiler = GeckoProfile(upload_dir, self.config, test)
def process_results(self):
# when running locally output results in build/raptor.json; when running
# in production output to a local.json to be turned into tc job artifact
if self.config.get('run_local', False):
if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
raptor_json_path = os.path.join(
os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing',
'mozharness', 'build', 'raptor.json')
else:
raptor_json_path = os.path.join(here, 'raptor.json')
else:
raptor_json_path = os.path.join(os.getcwd(), 'local.json')
self.config['raptor_json_path'] = raptor_json_path
return self.results_handler.summarize_and_output(self.config)
def get_page_timeout_list(self):
return self.results_handler.page_timeout_list
def clean_up(self):
self.control_server.stop()
if self.config['app'] != "geckoview":
self.runner.stop()
elif self.config['app'] == 'geckoview':
self.log.info('removing reverse socket connections')
self.device.remove_socket_connections('reverse')
else:
pass
self.log.info("finished")
class AbstractAndroidFirefox(object):
def __init__(self, certutil, binary):
self.proxy = None
self.certutil = certutil
self.app_args = [
"--marionette",
"--es",
"env0",
"LOG_VERBOSE=1",
"--es",
"env1",
"R_LOG_LEVEL=6",
]
self.binary = binary
self.skip_install = False
self.profile = None
def set_profile(self):
self.profile = create_profile("firefox")
print("Created profile: {}".format(self.profile.profile))
def create_certificate(self):
certdb = "sql:{}/".format(self.profile.profile)
print("Creating certificate database")
command = [self.certutil, "-N", "-v", "-d", certdb, "--empty-password"]
subprocess.call(command)
# install mitmproxy certificate
command = [
self.certutil,
"-A",
"-d",
certdb,
"-n",
"mitmproxy-cert",
"-t",
"TC,,",
"-a",
"-i",
self.proxy.cert,
]
print("Installing {} into certificate database".format(
self.proxy.cert))
subprocess.call(command)
command = [self.certutil, "-d", certdb, "-L"]
assert "mitmproxy-cert" in subprocess.check_output(command)
def setup_device(self):
self.device = ADBAndroid()
if self.binary and self.proxy.mode is "record":
if not self.skip_install:
if self.device.is_app_installed(self.APP_NAME):
print("Uninstalling app %s" % self.APP_NAME)
self.device.uninstall_app(self.APP_NAME)
print("Installing app %s" % self.APP_NAME)
self.device.install_app(apk_path=self.binary)
self.skip_install = True
else:
print("App already installed in a previous recording!!!!")
else:
print(
"No binary provided or proxy in replay mode! Using existing app on the device."
)
def setup_app(self):
self.device.shell("pm clear {}".format(self.APP_NAME))
self.device.create_socket_connection("reverse", "tcp:8080", "tcp:8080")
device_storage = "/sdcard/raptor"
device_profile = os.path.join(device_storage, "profile")
if self.device.is_dir(device_storage):
self.device.rm(device_storage, recursive=True)
self.device.mkdir(device_storage)
self.device.mkdir(device_profile)
self.app_args.extend(["-profile", device_profile])
userjs = os.path.join(self.profile.profile, "user.js")
with open(userjs) as f:
prefs = f.readlines()
prefs = [p for p in prefs if "network.proxy" not in p]
with open(userjs, "w") as f:
f.writelines(prefs)
self.profile.set_preferences({
"network.proxy.type":
1,
"network.proxy.http":
"127.0.0.1",
"network.proxy.http_port":
8080,
"network.proxy.ssl":
"127.0.0.1",
"network.proxy.ssl_port":
8080,
"network.proxy.no_proxies_on":
"localhost, 127.0.0.1",
})
self.device.push(self.profile.profile, device_profile)
self.device.chmod(device_storage, recursive=True)
def run_android_app(self, url):
raise NotImplementedError
def start(self, url="about:blank", proxy_service=None):
self.proxy = proxy_service
self.setup_device()
self.set_profile()
self.create_certificate()
self.setup_app()
self.run_android_app(url)
def stop(self):
self.device.stop_application(self.APP_NAME)
def screen_shot(self, path):
try:
self.device.rm("/sdcard/screen.png")
except ADBProcessError as e:
pass
self.device.shell("screencap -p /sdcard/screen.png")
self.device.pull("/sdcard/screen.png", path)
self.device.rm("/sdcard/screen.png")
def app_information(self):
if self.binary:
return mozversion.get_version(binary=self.binary)
return None
class MobileApp():
def __init__(self, config):
self.config = config
self.log = get_default_logger(component='recorder-mobile-app - ')
self.profile = self.config['profile']
def setup_app(self):
# create the android device handler; it gets initiated and sets up adb etc
self.log.info("creating android device handler using mozdevice")
self.device = ADBAndroid(verbose=True, logger_name="recorder-adb - ")
self.device_raptor_dir = "/sdcard/raptor"
self.config['device_raptor_dir'] = self.device_raptor_dir
if self.device.is_dir(self.device_raptor_dir):
self.log.info("deleting existing device raptor dir: %s" %
self.device_raptor_dir)
self.device.rm(self.device_raptor_dir, recursive=True)
self.log.info("creating raptor folder on sdcard: %s" %
self.device_raptor_dir)
self.device.mkdir(self.device_raptor_dir)
self.device.chmod(self.device_raptor_dir, recursive=True)
def start_app(self):
self.log.info("Adb Port redirect:")
_tcp_port = "tcp:8080"
self.device.create_socket_connection('reverse', _tcp_port, _tcp_port)
userjspath = os.path.join(self.profile.profile, 'user.js')
with open(userjspath) as userjsfile:
prefs = userjsfile.readlines()
prefs = [pref for pref in prefs if 'network.proxy' not in pref]
with open(userjspath, 'w') as userjsfile:
userjsfile.writelines(prefs)
no_proxies_on = "localhost, 127.0.0.1, %s" % self.config['host']
proxy_prefs = {}
proxy_prefs["network.proxy.type"] = 1
proxy_prefs["network.proxy.http"] = self.config['host']
proxy_prefs["network.proxy.http_port"] = 8080
proxy_prefs["network.proxy.ssl"] = self.config['host']
proxy_prefs["network.proxy.ssl_port"] = 8080
proxy_prefs["network.proxy.no_proxies_on"] = no_proxies_on
self.profile.set_preferences(proxy_prefs)
self.device_profile = os.path.join(self.device_raptor_dir, "profile")
if self.device.is_dir(self.device_profile):
self.log.info("deleting existing device profile folder: %s" %
self.device_profile)
self.device.rm(self.device_profile, recursive=True)
self.log.info("creating profile folder on device: %s" %
self.device_profile)
self.device.mkdir(self.device_profile)
self.log.info("copying firefox profile onto the device")
self.log.info("note: the profile folder being copied is: %s" %
self.profile.profile)
self.log.info(
'the adb push cmd copies that profile dir to a new temp dir before copy'
)
self.device.push(self.profile.profile, self.device_profile)
self.device.chmod(self.device_profile, recursive=True)
# now start the geckoview/fennec app
self.log.info("starting %s" % self.config['app'])
extra_args = [
"-profile",
self.device_profile,
"--marionette",
"--es",
"env0",
"LOG_VERBOSE=1",
"--es",
"env1",
"R_LOG_LEVEL=6",
]
# launch geckoview example app
try:
# make sure the geckoview app is not running before
# attempting to start.
self.device.stop_application(self.config['binary'])
self.device.launch_activity(self.config['binary'],
"GeckoViewActivity",
extra_args=extra_args,
url='about:blank',
e10s=True,
fail_if_running=False)
except Exception:
self.log.error("Exception launching %s" % self.config['binary'])
raise
class RobocopTestRunner(MochitestDesktop):
"""
A test harness for Robocop. Robocop tests are UI tests for Firefox for Android,
based on the Robotium test framework. This harness leverages some functionality
from mochitest, for convenience.
"""
# Some robocop tests run for >60 seconds without generating any output.
NO_OUTPUT_TIMEOUT = 180
def __init__(self, options, message_logger):
"""
Simple one-time initialization.
"""
MochitestDesktop.__init__(self, options.flavor, vars(options))
verbose = False
if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug':
verbose = True
self.device = ADBAndroid(adb=options.adbPath or 'adb',
device=options.deviceSerial,
test_root=options.remoteTestRoot,
verbose=verbose)
# Check that Firefox is installed
expected = options.app.split('/')[-1]
if not self.device.is_app_installed(expected):
raise Exception("%s is not installed on this device" % expected)
options.logFile = "robocop.log"
if options.remoteTestRoot is None:
options.remoteTestRoot = self.device.test_root
self.remoteProfile = posixpath.join(options.remoteTestRoot, "profile")
self.remoteProfileCopy = posixpath.join(options.remoteTestRoot,
"profile-copy")
self.remoteModulesDir = posixpath.join(options.remoteTestRoot,
"modules/")
self.remoteConfigFile = posixpath.join(options.remoteTestRoot,
"robotium.config")
self.remoteLogFile = posixpath.join(options.remoteTestRoot, "logs",
"robocop.log")
self.options = options
process_args = {'messageLogger': message_logger}
self.auto = RemoteAutomation(self.device,
options.remoteappname,
self.remoteProfile,
self.remoteLogFile,
processArgs=process_args)
self.environment = self.auto.environment
self.remoteScreenshots = "/mnt/sdcard/Robotium-Screenshots"
self.remoteMozLog = posixpath.join(options.remoteTestRoot, "mozlog")
self.localLog = options.logFile
self.localProfile = None
self.certdbNew = True
self.passed = 0
self.failed = 0
self.todo = 0
def startup(self):
"""
Second-stage initialization: One-time initialization which may require cleanup.
"""
# Despite our efforts to clean up servers started by this script, in practice
# we still see infrequent cases where a process is orphaned and interferes
# with future tests, typically because the old server is keeping the port in use.
# Try to avoid those failures by checking for and killing servers before
# trying to start new ones.
self.killNamedProc('ssltunnel')
self.killNamedProc('xpcshell')
self.auto.deleteANRs()
self.auto.deleteTombstones()
procName = self.options.app.split('/')[-1]
self.device.stop_application(procName)
if self.device.process_exist(procName):
self.log.warning("unable to kill %s before running tests!" %
procName)
self.device.rm(self.remoteScreenshots, force=True, recursive=True)
self.device.rm(self.remoteMozLog, force=True, recursive=True)
self.device.mkdir(self.remoteMozLog)
logParent = posixpath.dirname(self.remoteLogFile)
self.device.rm(logParent, force=True, recursive=True)
self.device.mkdir(logParent)
# Add Android version (SDK level) to mozinfo so that manifest entries
# can be conditional on android_version.
self.log.info(
"Android sdk version '%s'; will use this to filter manifests" %
str(self.device.version))
mozinfo.info['android_version'] = str(self.device.version)
if self.options.robocopApk:
self.device.install_app(self.options.robocopApk, replace=True)
self.log.debug("Robocop APK %s installed" %
self.options.robocopApk)
# Display remote diagnostics; if running in mach, keep output terse.
if self.options.log_mach is None:
self.printDeviceInfo()
self.setupLocalPaths()
self.buildProfile()
# ignoreSSLTunnelExts is a workaround for bug 1109310
self.startServers(self.options,
debuggerInfo=None,
ignoreSSLTunnelExts=True)
self.log.debug("Servers started")
def cleanup(self):
"""
Cleanup at end of job run.
"""
self.log.debug("Cleaning up...")
self.stopServers()
self.device.stop_application(self.options.app.split('/')[-1])
uploadDir = os.environ.get('MOZ_UPLOAD_DIR', None)
if uploadDir:
self.log.debug(
"Pulling any remote moz logs and screenshots to %s." %
uploadDir)
if self.device.is_dir(self.remoteMozLog):
self.device.pull(self.remoteMozLog, uploadDir)
if self.device.is_dir(self.remoteScreenshots):
self.device.pull(self.remoteScreenshots, uploadDir)
MochitestDesktop.cleanup(self, self.options)
if self.localProfile:
mozfile.remove(self.localProfile)
self.device.rm(self.remoteProfile, force=True, recursive=True)
self.device.rm(self.remoteProfileCopy, force=True, recursive=True)
self.device.rm(self.remoteScreenshots, force=True, recursive=True)
self.device.rm(self.remoteMozLog, force=True, recursive=True)
self.device.rm(self.remoteConfigFile, force=True)
self.device.rm(self.remoteLogFile, force=True)
self.log.debug("Cleanup complete.")
def findPath(self, paths, filename=None):
for path in paths:
p = path
if filename:
p = os.path.join(p, filename)
if os.path.exists(self.getFullPath(p)):
return path
return None
def makeLocalAutomation(self):
localAutomation = Automation()
localAutomation.IS_WIN32 = False
localAutomation.IS_LINUX = False
localAutomation.IS_MAC = False
localAutomation.UNIXISH = False
hostos = sys.platform
if (hostos == 'mac' or hostos == 'darwin'):
localAutomation.IS_MAC = True
elif (hostos == 'linux' or hostos == 'linux2'):
localAutomation.IS_LINUX = True
localAutomation.UNIXISH = True
elif (hostos == 'win32' or hostos == 'win64'):
localAutomation.BIN_SUFFIX = ".exe"
localAutomation.IS_WIN32 = True
return localAutomation
def setupLocalPaths(self):
"""
Setup xrePath and utilityPath and verify xpcshell.
This is similar to switchToLocalPaths in runtestsremote.py.
"""
localAutomation = self.makeLocalAutomation()
paths = [self.options.xrePath, localAutomation.DIST_BIN]
self.options.xrePath = self.findPath(paths)
if self.options.xrePath is None:
self.log.error(
"unable to find xulrunner path for %s, please specify with --xre-path"
% os.name)
sys.exit(1)
self.log.debug("using xre path %s" % self.options.xrePath)
xpcshell = "xpcshell"
if (os.name == "nt"):
xpcshell += ".exe"
if self.options.utilityPath:
paths = [self.options.utilityPath, self.options.xrePath]
else:
paths = [self.options.xrePath]
self.options.utilityPath = self.findPath(paths, xpcshell)
if self.options.utilityPath is None:
self.log.error(
"unable to find utility path for %s, please specify with --utility-path"
% os.name)
sys.exit(1)
self.log.debug("using utility path %s" % self.options.utilityPath)
xpcshell_path = os.path.join(self.options.utilityPath, xpcshell)
if localAutomation.elf_arm(xpcshell_path):
self.log.error('xpcshell at %s is an ARM binary; please use '
'the --utility-path argument to specify the path '
'to a desktop version.' % xpcshell_path)
sys.exit(1)
self.log.debug("xpcshell found at %s" % xpcshell_path)
def buildProfile(self):
"""
Build a profile locally, keep it locally for use by servers and
push a copy to the remote profile-copy directory.
This is similar to buildProfile in runtestsremote.py.
"""
self.options.extraPrefs.append('browser.search.suggest.enabled=true')
self.options.extraPrefs.append('browser.search.suggest.prompted=true')
self.options.extraPrefs.append('layout.css.devPixelsPerPx=1.0')
self.options.extraPrefs.append('browser.chrome.dynamictoolbar=false')
self.options.extraPrefs.append('browser.snippets.enabled=false')
self.options.extraPrefs.append('extensions.autoupdate.enabled=false')
# Override the telemetry init delay for integration testing.
self.options.extraPrefs.append('toolkit.telemetry.initDelay=1')
self.options.extensionsToExclude.extend([
'*****@*****.**',
])
self.extraPrefs = parse_preferences(self.options.extraPrefs)
if self.options.testingModulesDir:
try:
self.device.push(self.options.testingModulesDir,
self.remoteModulesDir)
self.device.chmod(self.remoteModulesDir,
recursive=True,
root=True)
except Exception:
self.log.error(
"Automation Error: Unable to copy test modules to device.")
raise
savedTestingModulesDir = self.options.testingModulesDir
self.options.testingModulesDir = self.remoteModulesDir
else:
savedTestingModulesDir = None
manifest = MochitestDesktop.buildProfile(self, self.options)
if savedTestingModulesDir:
self.options.testingModulesDir = savedTestingModulesDir
self.localProfile = self.options.profilePath
self.log.debug("Profile created at %s" % self.localProfile)
# some files are not needed for robocop; save time by not pushing
os.remove(os.path.join(self.localProfile, 'userChrome.css'))
try:
self.device.push(self.localProfile, self.remoteProfileCopy)
except Exception:
self.log.error(
"Automation Error: Unable to copy profile to device.")
raise
return manifest
def setupRemoteProfile(self):
"""
Remove any remote profile and re-create it.
"""
self.log.debug("Updating remote profile at %s" % self.remoteProfile)
self.device.rm(self.remoteProfile, force=True, recursive=True)
self.device.cp(self.remoteProfileCopy,
self.remoteProfile,
recursive=True)
def parseLocalLog(self):
"""
Read and parse the local log file, noting any failures.
"""
with open(self.localLog) as currentLog:
data = currentLog.readlines()
os.unlink(self.localLog)
start_found = False
end_found = False
fail_found = False
for line in data:
try:
message = json.loads(line)
if not isinstance(message, dict) or 'action' not in message:
continue
except ValueError:
continue
if message['action'] == 'test_end':
end_found = True
start_found = False
break
if start_found and not end_found:
if 'status' in message:
if 'expected' in message:
self.failed += 1
elif message['status'] == 'PASS':
self.passed += 1
elif message['status'] == 'FAIL':
self.todo += 1
if message['action'] == 'test_start':
start_found = True
if 'expected' in message:
fail_found = True
result = 0
if fail_found:
result = 1
if not end_found:
self.log.info(
"PROCESS-CRASH | Automation Error: Missing end of test marker (process crashed?)"
)
result = 1
return result
def logTestSummary(self):
"""
Print a summary of all tests run to stdout, for treeherder parsing
(logging via self.log does not work here).
"""
print("0 INFO TEST-START | Shutdown")
print("1 INFO Passed: %s" % (self.passed))
print("2 INFO Failed: %s" % (self.failed))
print("3 INFO Todo: %s" % (self.todo))
print("4 INFO SimpleTest FINISHED")
if self.failed > 0:
return 1
return 0
def printDeviceInfo(self, printLogcat=False):
"""
Log remote device information and logcat (if requested).
This is similar to printDeviceInfo in runtestsremote.py
"""
try:
if printLogcat:
logcat = self.device.get_logcat(
filter_out_regexps=fennecLogcatFilters)
for l in logcat:
self.log.info(l.decode('utf-8', 'replace'))
self.log.info("Device info:")
devinfo = self.device.get_info()
for category in devinfo:
if type(devinfo[category]) is list:
self.log.info(" %s:" % category)
for item in devinfo[category]:
self.log.info(" %s" % item)
else:
self.log.info(" %s: %s" % (category, devinfo[category]))
self.log.info("Test root: %s" % self.device.test_root)
except ADBTimeoutError:
raise
except Exception as e:
self.log.warning("Error getting device information: %s" % str(e))
def setupRobotiumConfig(self, browserEnv):
"""
Create robotium.config and push it to the device.
"""
fHandle = tempfile.NamedTemporaryFile(suffix='.config',
prefix='robotium-',
dir=os.getcwd(),
delete=False)
fHandle.write("profile=%s\n" % self.remoteProfile)
fHandle.write("logfile=%s\n" % self.remoteLogFile)
fHandle.write("host=http://mochi.test:8888/tests\n")
fHandle.write("rawhost=http://%s:%s/tests\n" %
(self.options.remoteWebServer, self.options.httpPort))
if browserEnv:
envstr = ""
delim = ""
for key, value in browserEnv.items():
try:
value.index(',')
self.log.error(
"setupRobotiumConfig: browserEnv - Found a ',' "
"in our value, unable to process value. key=%s,value=%s"
% (key, value))
self.log.error("browserEnv=%s" % browserEnv)
except ValueError:
envstr += "%s%s=%s" % (delim, key, value)
delim = ","
fHandle.write("envvars=%s\n" % envstr)
fHandle.close()
self.device.rm(self.remoteConfigFile, force=True)
self.device.push(fHandle.name, self.remoteConfigFile)
os.unlink(fHandle.name)
def buildBrowserEnv(self):
"""
Return an environment dictionary suitable for remote use.
This is similar to buildBrowserEnv in runtestsremote.py.
"""
browserEnv = self.environment(xrePath=None, debugger=None)
# remove desktop environment not used on device
if "XPCOM_MEM_BLOAT_LOG" in browserEnv:
del browserEnv["XPCOM_MEM_BLOAT_LOG"]
browserEnv["MOZ_LOG_FILE"] = os.path.join(self.remoteMozLog,
self.mozLogName)
try:
browserEnv.update(
dict(
parse_key_value(self.options.environment,
context='--setenv')))
except KeyValueParseError as e:
self.log.error(str(e))
return None
return browserEnv
def runSingleTest(self, test):
"""
Run the specified test.
"""
self.log.debug("Running test %s" % test['name'])
self.mozLogName = "moz-%s.log" % test['name']
browserEnv = self.buildBrowserEnv()
self.setupRobotiumConfig(browserEnv)
self.setupRemoteProfile()
self.options.app = "am"
timeout = None
testName = test['name'].split('/')[-1].split('.java')[0]
if self.options.enable_coverage:
remoteCoverageFile = posixpath.join(
self.options.remoteTestRoot,
'robocop-coverage-%s.ec' % testName)
coverageFile = os.path.join(self.options.coverage_output_dir,
'robocop-coverage-%s.ec' % testName)
if self.options.autorun:
# This launches a test (using "am instrument") and instructs
# Fennec to /quit/ the browser (using Robocop:Quit) and to
# /finish/ all opened activities.
browserArgs = [
"instrument",
]
if self.options.enable_coverage:
browserArgs += [
"-e",
"coverage",
"true",
"-e",
"coverageFile",
remoteCoverageFile,
]
browserArgs += [
"-e",
"quit_and_finish",
"1",
"-e",
"deviceroot",
self.device.test_root,
"-e",
"class",
"org.mozilla.gecko.tests.%s" % testName,
"org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner",
]
else:
# This does not launch a test at all. It launches an activity
# that starts Fennec and then waits indefinitely, since cat
# never returns.
browserArgs = [
"start", "-n", "org.mozilla.roboexample.test/org.mozilla."
"gecko.LaunchFennecWithConfigurationActivity", "&&", "cat"
]
timeout = sys.maxint # Forever.
self.log.info("")
self.log.info(
"Serving mochi.test Robocop root at http://%s:%s/tests/robocop/"
% (self.options.remoteWebServer, self.options.httpPort))
self.log.info("")
result = -1
log_result = -1
try:
self.device.clear_logcat()
if not timeout:
timeout = self.options.timeout
if not timeout:
timeout = self.NO_OUTPUT_TIMEOUT
result, _ = self.auto.runApp(None,
browserEnv,
"am",
self.localProfile,
browserArgs,
timeout=timeout,
symbolsPath=self.options.symbolsPath)
self.log.debug("runApp completes with status %d" % result)
if result != 0:
self.log.error("runApp() exited with code %s" % result)
if self.device.is_file(self.remoteLogFile):
self.device.pull(self.remoteLogFile, self.localLog)
self.device.rm(self.remoteLogFile)
log_result = self.parseLocalLog()
if result != 0 or log_result != 0:
# Display remote diagnostics; if running in mach, keep output
# terse.
if self.options.log_mach is None:
self.printDeviceInfo(printLogcat=True)
if self.options.enable_coverage:
if self.device.is_file(remoteCoverageFile):
self.device.pull(remoteCoverageFile, coverageFile)
self.device.rm(remoteCoverageFile)
else:
self.log.warning(
"Code coverage output not found on remote device: %s" %
remoteCoverageFile)
except Exception:
self.log.error(
"Automation Error: Exception caught while running tests")
traceback.print_exc()
result = 1
self.log.debug("Test %s completes with status %d (log status %d)" %
(test['name'], int(result), int(log_result)))
return result
def runTests(self):
self.startup()
if isinstance(self.options.manifestFile, TestManifest):
mp = self.options.manifestFile
else:
mp = TestManifest(strict=False)
mp.read("robocop.ini")
filters = []
if self.options.totalChunks:
filters.append(
chunk_by_slice(self.options.thisChunk,
self.options.totalChunks))
robocop_tests = mp.active_tests(exists=False,
filters=filters,
**mozinfo.info)
if not self.options.autorun:
# Force a single loop iteration. The iteration will start Fennec and
# the httpd server, but not actually run a test.
self.options.test_paths = [robocop_tests[0]['name']]
active_tests = []
for test in robocop_tests:
if self.options.test_paths and test[
'name'] not in self.options.test_paths:
continue
if 'disabled' in test:
self.log.info('TEST-INFO | skipping %s | %s' %
(test['name'], test['disabled']))
continue
active_tests.append(test)
tests_by_manifest = defaultdict(list)
for test in active_tests:
tests_by_manifest[test['manifest']].append(test['name'])
self.log.suite_start(tests_by_manifest)
worstTestResult = None
for test in active_tests:
result = self.runSingleTest(test)
if worstTestResult is None or worstTestResult == 0:
worstTestResult = result
if worstTestResult is None:
self.log.warning(
"No tests run. Did you pass an invalid TEST_PATH?")
worstTestResult = 1
else:
print "INFO | runtests.py | Test summary: start."
logResult = self.logTestSummary()
print "INFO | runtests.py | Test summary: end."
if worstTestResult == 0:
worstTestResult = logResult
return worstTestResult
def start(self, url="about:blank"):
# create profile
profile = create_profile("firefox")
print("Created profile: {}".format(profile.profile))
# create certificate database
certdb = "sql:{}/".format(profile.profile)
print("Creating certificate database")
command = [self.certutil, "-N", "-v", "-d", certdb, "--empty-password"]
subprocess.call(command)
# install mitmproxy certificate
command = [
self.certutil,
"-A",
"-d",
certdb,
"-n",
"mitmproxy-cert",
"-t",
"TC,,",
"-a",
"-i",
self.proxy.cert,
]
print("Installing {} into certificate database".format(
self.proxy.cert))
subprocess.call(command)
# verify certificate is installed
command = [self.certutil, "-d", certdb, "-L"]
assert "mitmproxy-cert" in subprocess.check_output(command)
# setup device
device = ADBAndroid()
device.shell("pm clear {}".format(self.APP_NAME))
device.create_socket_connection("reverse", "tcp:8080", "tcp:8080")
device_storage = "/sdcard/raptor"
device_profile = os.path.join(device_storage, "profile")
if device.is_dir(device_storage):
device.rm(device_storage, recursive=True)
device.mkdir(device_storage)
device.mkdir(device_profile)
userjs = os.path.join(profile.profile, "user.js")
with open(userjs) as f:
prefs = f.readlines()
prefs = [p for p in prefs if "network.proxy" not in p]
with open(userjs, "w") as f:
f.writelines(prefs)
profile.set_preferences({
"network.proxy.type":
1,
"network.proxy.http":
"127.0.0.1",
"network.proxy.http_port":
8080,
"network.proxy.ssl":
"127.0.0.1",
"network.proxy.ssl_port":
8080,
"network.proxy.no_proxies_on":
"localhost, 127.0.0.1",
})
device.push(profile.profile, device_profile)
device.chmod(device_storage, recursive=True)
app_args = [
"-profile",
device_profile,
"--marionette",
"--es",
"env0",
"LOG_VERBOSE=1",
"--es",
"env1",
"R_LOG_LEVEL=6",
]
# start app
device.stop_application(self.APP_NAME)
device.launch_activity(
self.APP_NAME,
self.ACTIVITY_NAME,
extra_args=app_args,
url=url,
e10s=True,
fail_if_running=False,
)
