def run_tests_remote(tests, num_tests, prefix, options, slog):
# Setup device with everything needed to run our tests.
from mozdevice import ADBDevice, ADBError, ADBTimeoutError
try:
device = ADBDevice(device=options.device_serial,
test_root=options.remote_test_root)
init_remote_dir(device, options.remote_test_root)
# Update the test root to point to our test directory.
jit_tests_dir = posixpath.join(options.remote_test_root, 'jit-tests')
options.remote_test_root = posixpath.join(jit_tests_dir, 'jit-tests')
# Push js shell and libraries.
init_remote_dir(device, jit_tests_dir)
push_libs(options, device)
push_progs(options, device, [prefix[0]])
device.chmod(options.remote_test_root, recursive=True, root=True)
jtd_tests = posixpath.join(jit_tests_dir, 'tests')
init_remote_dir(device, jtd_tests)
device.push(JS_TESTS_DIR, jtd_tests, timeout=600)
device.chmod(jtd_tests, recursive=True, root=True)
device.push(os.path.dirname(TEST_DIR),
options.remote_test_root,
timeout=600)
device.chmod(options.remote_test_root, recursive=True, root=True)
prefix[0] = os.path.join(options.remote_test_root, 'js')
except (ADBError, ADBTimeoutError):
print("TEST-UNEXPECTED-FAIL | jit_test.py" +
" : Device initialization failed")
raise
# Run all tests.
pb = create_progressbar(num_tests, options)
try:
gen = get_remote_results(tests, device, prefix, options)
ok = process_test_results(gen, num_tests, pb, options, slog)
except (ADBError, ADBTimeoutError):
print("TEST-UNEXPECTED-FAIL | jit_test.py" +
" : Device error during test")
raise
return ok
def run_tests_remote(tests, num_tests, prefix, options, slog):
# Setup device with everything needed to run our tests.
from mozdevice import ADBDevice
device = ADBDevice(device=options.device_serial,
test_root=options.remote_test_root)
init_remote_dir(device, options.remote_test_root)
# Update the test root to point to our test directory.
jit_tests_dir = posixpath.join(options.remote_test_root, 'jit-tests')
options.remote_test_root = posixpath.join(jit_tests_dir, 'jit-tests')
# Push js shell and libraries.
init_remote_dir(device, jit_tests_dir)
push_libs(options, device)
push_progs(options, device, [prefix[0]])
device.chmod(options.remote_test_root, recursive=True, root=True)
JitTest.CacheDir = posixpath.join(options.remote_test_root, '.js-cache')
init_remote_dir(device, JitTest.CacheDir)
jtd_tests = posixpath.join(jit_tests_dir, 'tests')
init_remote_dir(device, jtd_tests)
device.push(JS_TESTS_DIR, jtd_tests, timeout=600)
device.chmod(jtd_tests, recursive=True, root=True)
device.push(os.path.dirname(TEST_DIR),
options.remote_test_root,
timeout=600)
device.chmod(options.remote_test_root, recursive=True, root=True)
prefix[0] = os.path.join(options.remote_test_root, 'js')
# Run all tests.
pb = create_progressbar(num_tests, options)
gen = get_remote_results(tests, device, prefix, options)
ok = process_test_results(gen, num_tests, pb, options, slog)
return ok
def run_tests_remote(tests, num_tests, prefix, options, slog):
# Setup device with everything needed to run our tests.
from mozdevice import ADBDevice
device = ADBDevice(device=options.device_serial,
test_root=options.remote_test_root)
init_remote_dir(device, options.remote_test_root)
# Update the test root to point to our test directory.
jit_tests_dir = posixpath.join(options.remote_test_root, 'jit-tests')
options.remote_test_root = posixpath.join(jit_tests_dir, 'jit-tests')
# Push js shell and libraries.
init_remote_dir(device, jit_tests_dir)
push_libs(options, device)
push_progs(options, device, [prefix[0]])
device.chmod(options.remote_test_root, recursive=True, root=True)
JitTest.CacheDir = posixpath.join(options.remote_test_root, '.js-cache')
init_remote_dir(device, JitTest.CacheDir)
jtd_tests = posixpath.join(jit_tests_dir, 'tests')
init_remote_dir(device, jtd_tests)
device.push(JS_TESTS_DIR, jtd_tests, timeout=600)
device.chmod(jtd_tests, recursive=True, root=True)
device.push(os.path.dirname(TEST_DIR), options.remote_test_root,
timeout=600)
device.chmod(options.remote_test_root, recursive=True, root=True)
prefix[0] = os.path.join(options.remote_test_root, 'js')
# Run all tests.
pb = create_progressbar(num_tests, options)
gen = get_remote_results(tests, device, prefix, options)
ok = process_test_results(gen, num_tests, pb, options, slog)
return ok
class MochiRemote(MochitestDesktop):
localProfile = None
logMessages = []
def __init__(self, options):
MochitestDesktop.__init__(self, options.flavor, vars(options))
verbose = False
if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug':
verbose = True
if hasattr(options, 'log'):
delattr(options, 'log')
self.certdbNew = True
self.chromePushed = False
self.mozLogName = "moz.log"
self.device = ADBDevice(adb=options.adbPath or 'adb',
device=options.deviceSerial,
test_root=options.remoteTestRoot,
verbose=verbose)
if options.remoteTestRoot is None:
options.remoteTestRoot = self.device.test_root
options.dumpOutputDirectory = options.remoteTestRoot
self.remoteLogFile = posixpath.join(options.remoteTestRoot, "logs", "mochitest.log")
logParent = posixpath.dirname(self.remoteLogFile)
self.device.rm(logParent, force=True, recursive=True)
self.device.mkdir(logParent)
self.remoteProfile = posixpath.join(options.remoteTestRoot, "profile/")
self.device.rm(self.remoteProfile, force=True, recursive=True)
self.counts = dict()
self.message_logger = MessageLogger(logger=None)
self.message_logger.logger = self.log
process_args = {'messageLogger': self.message_logger, 'counts': self.counts}
self.automation = RemoteAutomation(self.device, options.remoteappname, self.remoteProfile,
self.remoteLogFile, processArgs=process_args)
self.environment = self.automation.environment
# Check that Firefox is installed
expected = options.app.split('/')[-1]
if not self.device.is_app_installed(expected):
raise Exception("%s is not installed on this device" % expected)
self.automation.deleteANRs()
self.automation.deleteTombstones()
self.device.clear_logcat()
self.remoteModulesDir = posixpath.join(options.remoteTestRoot, "modules/")
self.remoteCache = posixpath.join(options.remoteTestRoot, "cache/")
self.device.rm(self.remoteCache, force=True, recursive=True)
# move necko cache to a location that can be cleaned up
options.extraPrefs += ["browser.cache.disk.parent_directory=%s" % self.remoteCache]
self.remoteMozLog = posixpath.join(options.remoteTestRoot, "mozlog")
self.device.rm(self.remoteMozLog, force=True, recursive=True)
self.device.mkdir(self.remoteMozLog)
self.remoteChromeTestDir = posixpath.join(
options.remoteTestRoot,
"chrome")
self.device.rm(self.remoteChromeTestDir, force=True, recursive=True)
self.device.mkdir(self.remoteChromeTestDir)
procName = options.app.split('/')[-1]
self.device.stop_application(procName)
if self.device.process_exist(procName):
self.log.warning("unable to kill %s before running tests!" % procName)
# Add Android version (SDK level) to mozinfo so that manifest entries
# can be conditional on android_version.
self.log.info(
"Android sdk version '%s'; will use this to filter manifests" %
str(self.device.version))
mozinfo.info['android_version'] = str(self.device.version)
mozinfo.info['isFennec'] = not ('geckoview' in options.app)
mozinfo.info['is_emulator'] = self.device._device_serial.startswith('emulator-')
def cleanup(self, options, final=False):
if final:
self.device.rm(self.remoteChromeTestDir, force=True, recursive=True)
self.chromePushed = False
uploadDir = os.environ.get('MOZ_UPLOAD_DIR', None)
if uploadDir and self.device.is_dir(self.remoteMozLog):
self.device.pull(self.remoteMozLog, uploadDir)
self.device.rm(self.remoteLogFile, force=True)
self.device.rm(self.remoteProfile, force=True, recursive=True)
self.device.rm(self.remoteCache, force=True, recursive=True)
MochitestDesktop.cleanup(self, options, final)
self.localProfile = None
def dumpScreen(self, utilityPath):
if self.haveDumpedScreen:
self.log.info(
"Not taking screenshot here: see the one that was previously logged")
return
self.haveDumpedScreen = True
if self.device._device_serial.startswith('emulator-'):
dump_screen(utilityPath, self.log)
else:
dump_device_screen(self.device, self.log)
def findPath(self, paths, filename=None):
for path in paths:
p = path
if filename:
p = os.path.join(p, filename)
if os.path.exists(self.getFullPath(p)):
return path
return None
def makeLocalAutomation(self):
localAutomation = Automation()
localAutomation.IS_WIN32 = False
localAutomation.IS_LINUX = False
localAutomation.IS_MAC = False
localAutomation.UNIXISH = False
hostos = sys.platform
if (hostos == 'mac' or hostos == 'darwin'):
localAutomation.IS_MAC = True
elif (hostos == 'linux' or hostos == 'linux2'):
localAutomation.IS_LINUX = True
localAutomation.UNIXISH = True
elif (hostos == 'win32' or hostos == 'win64'):
localAutomation.BIN_SUFFIX = ".exe"
localAutomation.IS_WIN32 = True
return localAutomation
# This seems kludgy, but this class uses paths from the remote host in the
# options, except when calling up to the base class, which doesn't
# understand the distinction. This switches out the remote values for local
# ones that the base class understands. This is necessary for the web
# server, SSL tunnel and profile building functions.
def switchToLocalPaths(self, options):
""" Set local paths in the options, return a function that will restore remote values """
remoteXrePath = options.xrePath
remoteProfilePath = options.profilePath
remoteUtilityPath = options.utilityPath
localAutomation = self.makeLocalAutomation()
paths = [
options.xrePath,
localAutomation.DIST_BIN,
]
options.xrePath = self.findPath(paths)
if options.xrePath is None:
self.log.error(
"unable to find xulrunner path for %s, please specify with --xre-path" %
os.name)
sys.exit(1)
xpcshell = "xpcshell"
if (os.name == "nt"):
xpcshell += ".exe"
if options.utilityPath:
paths = [options.utilityPath, options.xrePath]
else:
paths = [options.xrePath]
options.utilityPath = self.findPath(paths, xpcshell)
if options.utilityPath is None:
self.log.error(
"unable to find utility path for %s, please specify with --utility-path" %
os.name)
sys.exit(1)
xpcshell_path = os.path.join(options.utilityPath, xpcshell)
if localAutomation.elf_arm(xpcshell_path):
self.log.error('xpcshell at %s is an ARM binary; please use '
'the --utility-path argument to specify the path '
'to a desktop version.' % xpcshell_path)
sys.exit(1)
if self.localProfile:
options.profilePath = self.localProfile
else:
options.profilePath = None
def fixup():
options.xrePath = remoteXrePath
options.utilityPath = remoteUtilityPath
options.profilePath = remoteProfilePath
return fixup
def startServers(self, options, debuggerInfo):
""" Create the servers on the host and start them up """
restoreRemotePaths = self.switchToLocalPaths(options)
MochitestDesktop.startServers(
self,
options,
debuggerInfo)
restoreRemotePaths()
def buildProfile(self, options):
restoreRemotePaths = self.switchToLocalPaths(options)
if options.testingModulesDir:
try:
self.device.push(options.testingModulesDir, self.remoteModulesDir)
self.device.chmod(self.remoteModulesDir, recursive=True, root=True)
except Exception:
self.log.error(
"Automation Error: Unable to copy test modules to device.")
raise
savedTestingModulesDir = options.testingModulesDir
options.testingModulesDir = self.remoteModulesDir
else:
savedTestingModulesDir = None
manifest = MochitestDesktop.buildProfile(self, options)
if savedTestingModulesDir:
options.testingModulesDir = savedTestingModulesDir
self.localProfile = options.profilePath
restoreRemotePaths()
options.profilePath = self.remoteProfile
return manifest
def buildURLOptions(self, options, env):
saveLogFile = options.logFile
options.logFile = self.remoteLogFile
options.profilePath = self.localProfile
env["MOZ_HIDE_RESULTS_TABLE"] = "1"
retVal = MochitestDesktop.buildURLOptions(self, options, env)
# we really need testConfig.js (for browser chrome)
try:
self.device.push(options.profilePath, self.remoteProfile)
self.device.chmod(self.remoteProfile, recursive=True, root=True)
except Exception:
self.log.error("Automation Error: Unable to copy profile to device.")
raise
options.profilePath = self.remoteProfile
options.logFile = saveLogFile
return retVal
def getChromeTestDir(self, options):
local = super(MochiRemote, self).getChromeTestDir(options)
remote = self.remoteChromeTestDir
if options.flavor == 'chrome' and not self.chromePushed:
self.log.info("pushing %s to %s on device..." % (local, remote))
local = os.path.join(local, "chrome")
self.device.push(local, remote)
self.chromePushed = True
return remote
def getLogFilePath(self, logFile):
return logFile
def printDeviceInfo(self, printLogcat=False):
try:
if printLogcat:
logcat = self.device.get_logcat(
filter_out_regexps=fennecLogcatFilters)
for l in logcat:
ul = l.decode('utf-8', errors='replace')
sl = ul.encode('iso8859-1', errors='replace')
self.log.info(sl)
self.log.info("Device info:")
devinfo = self.device.get_info()
for category in devinfo:
if type(devinfo[category]) is list:
self.log.info(" %s:" % category)
for item in devinfo[category]:
self.log.info(" %s" % item)
else:
self.log.info(" %s: %s" % (category, devinfo[category]))
self.log.info("Test root: %s" % self.device.test_root)
except ADBTimeoutError:
raise
except Exception as e:
self.log.warning("Error getting device information: %s" % str(e))
def getGMPPluginPath(self, options):
# TODO: bug 1149374
return None
def buildBrowserEnv(self, options, debugger=False):
browserEnv = MochitestDesktop.buildBrowserEnv(
self,
options,
debugger=debugger)
# remove desktop environment not used on device
if "XPCOM_MEM_BLOAT_LOG" in browserEnv:
del browserEnv["XPCOM_MEM_BLOAT_LOG"]
# override mozLogs to avoid processing in MochitestDesktop base class
self.mozLogs = None
browserEnv["MOZ_LOG_FILE"] = os.path.join(
self.remoteMozLog,
self.mozLogName)
if options.dmd:
browserEnv['DMD'] = '1'
# Contents of remoteMozLog will be pulled from device and copied to the
# host MOZ_UPLOAD_DIR, to be made available as test artifacts. Make
# MOZ_UPLOAD_DIR available to the browser environment so that tests
# can use it as though they were running on the host.
browserEnv["MOZ_UPLOAD_DIR"] = self.remoteMozLog
return browserEnv
def runApp(self, *args, **kwargs):
"""front-end automation.py's `runApp` functionality until FennecRunner is written"""
# automation.py/remoteautomation `runApp` takes the profile path,
# whereas runtest.py's `runApp` takes a mozprofile object.
if 'profileDir' not in kwargs and 'profile' in kwargs:
kwargs['profileDir'] = kwargs.pop('profile').profile
# remove args not supported by automation.py
kwargs.pop('marionette_args', None)
ret, _ = self.automation.runApp(*args, **kwargs)
self.countpass += self.counts['pass']
self.countfail += self.counts['fail']
self.counttodo += self.counts['todo']
return ret, None
class XPCShellRemote(xpcshell.XPCShellTests, object):
def __init__(self, options, log):
xpcshell.XPCShellTests.__init__(self, log)
self.options = options
verbose = False
if options['log_tbpl_level'] == 'debug' or options['log_mach_level'] == 'debug':
verbose = True
self.device = ADBDevice(adb=options['adbPath'] or 'adb',
device=options['deviceSerial'],
test_root=options['remoteTestRoot'],
verbose=verbose)
self.remoteTestRoot = posixpath.join(self.device.test_root, "xpc")
# Add Android version (SDK level) to mozinfo so that manifest entries
# can be conditional on android_version.
mozinfo.info['android_version'] = self.device.version
self.localBin = options['localBin']
self.pathMapping = []
# remoteBinDir contains xpcshell and its wrapper script, both of which must
# be executable. Since +x permissions cannot usually be set on /mnt/sdcard,
# and the test root may be on /mnt/sdcard, remoteBinDir is set to be on
# /data/local, always.
self.remoteBinDir = posixpath.join("/data", "local", "xpcb")
# Terse directory names are used here ("c" for the components directory)
# to minimize the length of the command line used to execute
# xpcshell on the remote device. adb has a limit to the number
# of characters used in a shell command, and the xpcshell command
# line can be quite complex.
self.remoteTmpDir = posixpath.join(self.remoteTestRoot, "tmp")
self.remoteScriptsDir = self.remoteTestRoot
self.remoteComponentsDir = posixpath.join(self.remoteTestRoot, "c")
self.remoteModulesDir = posixpath.join(self.remoteTestRoot, "m")
self.remoteMinidumpDir = posixpath.join(self.remoteTestRoot, "minidumps")
self.profileDir = posixpath.join(self.remoteTestRoot, "p")
self.remoteDebugger = options['debugger']
self.remoteDebuggerArgs = options['debuggerArgs']
self.testingModulesDir = options['testingModulesDir']
self.env = {}
if options['objdir']:
self.xpcDir = os.path.join(options['objdir'], "_tests/xpcshell")
elif os.path.isdir(os.path.join(here, 'tests')):
self.xpcDir = os.path.join(here, 'tests')
else:
print("Couldn't find local xpcshell test directory", file=sys.stderr)
sys.exit(1)
self.localAPKContents = ZipFile(options['localAPK'])
if options['setup']:
self.setupTestDir()
self.setupUtilities()
self.setupModules()
self.initDir(self.remoteMinidumpDir)
self.remoteAPK = None
self.remoteAPK = posixpath.join(self.remoteBinDir,
os.path.basename(options['localAPK']))
self.setAppRoot()
# data that needs to be passed to the RemoteXPCShellTestThread
self.mobileArgs = {
'device': self.device,
'remoteBinDir': self.remoteBinDir,
'remoteScriptsDir': self.remoteScriptsDir,
'remoteComponentsDir': self.remoteComponentsDir,
'remoteModulesDir': self.remoteModulesDir,
'options': self.options,
'remoteDebugger': self.remoteDebugger,
'pathMapping': self.pathMapping,
'profileDir': self.profileDir,
'remoteTmpDir': self.remoteTmpDir,
'remoteMinidumpDir': self.remoteMinidumpDir,
}
if self.remoteAPK:
self.mobileArgs['remoteAPK'] = self.remoteAPK
def initDir(self, path, mask="777", timeout=None, root=True):
"""Initialize a directory by removing it if it exists, creating it
and changing the permissions."""
self.device.rm(path, recursive=True, force=True, timeout=timeout, root=root)
self.device.mkdir(path, parents=True, timeout=timeout, root=root)
self.device.chmod(path, recursive=True, mask=mask, timeout=timeout, root=root)
def setLD_LIBRARY_PATH(self):
self.env["LD_LIBRARY_PATH"] = self.remoteBinDir
def pushWrapper(self):
# Rather than executing xpcshell directly, this wrapper script is
# used. By setting environment variables and the cwd in the script,
# the length of the per-test command line is shortened. This is
# often important when using ADB, as there is a limit to the length
# of the ADB command line.
localWrapper = tempfile.mktemp()
f = open(localWrapper, "w")
f.write("#!/system/bin/sh\n")
for envkey, envval in self.env.iteritems():
f.write("export %s=%s\n" % (envkey, envval))
f.writelines([
"cd $1\n",
"echo xpcw: cd $1\n",
"shift\n",
"echo xpcw: xpcshell \"$@\"\n",
"%s/xpcshell \"$@\"\n" % self.remoteBinDir])
f.close()
remoteWrapper = posixpath.join(self.remoteBinDir, "xpcw")
self.device.push(localWrapper, remoteWrapper)
self.device.chmod(remoteWrapper, root=True)
os.remove(localWrapper)
def buildPrefsFile(self, extraPrefs):
prefs = super(XPCShellRemote, self).buildPrefsFile(extraPrefs)
remotePrefsFile = posixpath.join(self.remoteTestRoot, 'user.js')
self.device.push(self.prefsFile, remotePrefsFile)
self.device.chmod(remotePrefsFile, root=True)
os.remove(self.prefsFile)
self.prefsFile = remotePrefsFile
return prefs
def buildEnvironment(self):
self.buildCoreEnvironment()
self.setLD_LIBRARY_PATH()
self.env["MOZ_LINKER_CACHE"] = self.remoteBinDir
if self.appRoot:
self.env["GRE_HOME"] = self.appRoot
self.env["XPCSHELL_TEST_PROFILE_DIR"] = self.profileDir
self.env["TMPDIR"] = self.remoteTmpDir
self.env["HOME"] = self.profileDir
self.env["XPCSHELL_TEST_TEMP_DIR"] = self.remoteTmpDir
self.env["XPCSHELL_MINIDUMP_DIR"] = self.remoteMinidumpDir
self.env["MOZ_ANDROID_CPU_ABI"] = self.device.get_prop("ro.product.cpu.abi")
if self.options['setup']:
self.pushWrapper()
def setAppRoot(self):
# Determine the application root directory associated with the package
# name used by the APK.
self.appRoot = None
packageName = None
try:
packageName = self.localAPKContents.read("package-name.txt")
except Exception as e:
print("unable to determine app root; assuming geckoview: " + str(e))
packageName = "org.mozilla.geckoview.test"
if packageName:
self.appRoot = posixpath.join("/data", "data", packageName.strip())
def setupUtilities(self):
self.initDir(self.remoteTmpDir)
self.initDir(self.remoteBinDir)
remotePrefDir = posixpath.join(self.remoteBinDir, "defaults", "pref")
self.initDir(posixpath.join(remotePrefDir, "extra"))
self.initDir(self.remoteComponentsDir)
local = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'head.js')
remoteFile = posixpath.join(self.remoteScriptsDir, "head.js")
self.device.push(local, remoteFile)
self.device.chmod(remoteFile, root=True)
# The xpcshell binary is required for all tests. Additional binaries
# are required for some tests. This list should be similar to
# TEST_HARNESS_BINS in testing/mochitest/Makefile.in.
binaries = ["xpcshell",
"ssltunnel",
"certutil",
"pk12util",
"BadCertServer",
"OCSPStaplingServer",
"GenerateOCSPResponse",
"SymantecSanctionsServer"]
for fname in binaries:
local = os.path.join(self.localBin, fname)
if os.path.isfile(local):
print("Pushing %s.." % fname, file=sys.stderr)
remoteFile = posixpath.join(self.remoteBinDir, fname)
self.device.push(local, remoteFile)
self.device.chmod(remoteFile, root=True)
else:
print("*** Expected binary %s not found in %s!" %
(fname, self.localBin), file=sys.stderr)
local = os.path.join(self.localBin, "components/httpd.js")
remoteFile = posixpath.join(self.remoteComponentsDir, "httpd.js")
self.device.push(local, remoteFile)
self.device.chmod(remoteFile, root=True)
local = os.path.join(self.localBin, "components/httpd.manifest")
remoteFile = posixpath.join(self.remoteComponentsDir, "httpd.manifest")
self.device.push(local, remoteFile)
self.device.chmod(remoteFile, root=True)
remoteFile = posixpath.join(self.remoteBinDir,
os.path.basename(self.options['localAPK']))
self.device.push(self.options['localAPK'], remoteFile)
self.device.chmod(remoteFile, root=True)
self.pushLibs()
def pushLibs(self):
elfhack = os.path.join(self.localBin, 'elfhack')
if not os.path.exists(elfhack):
elfhack = None
pushed_libs_count = 0
try:
dir = tempfile.mkdtemp()
for info in self.localAPKContents.infolist():
if info.filename.endswith(".so"):
print("Pushing %s.." % info.filename, file=sys.stderr)
remoteFile = posixpath.join(self.remoteBinDir,
os.path.basename(info.filename))
self.localAPKContents.extract(info, dir)
localFile = os.path.join(dir, info.filename)
with open(localFile) as f:
# Decompress xz-compressed file.
if f.read(5)[1:] == '7zXZ':
cmd = ['xz', '-df', '--suffix', '.so', localFile]
subprocess.check_output(cmd)
# xz strips the ".so" file suffix.
os.rename(localFile[:-3], localFile)
# elfhack -r should provide better crash reports
if elfhack:
cmd = [elfhack, '-r', localFile]
subprocess.check_output(cmd)
self.device.push(localFile, remoteFile)
pushed_libs_count += 1
self.device.chmod(remoteFile, root=True)
finally:
shutil.rmtree(dir)
return pushed_libs_count
def setupModules(self):
if self.testingModulesDir:
self.device.push(self.testingModulesDir, self.remoteModulesDir)
self.device.chmod(self.remoteModulesDir, root=True)
def setupTestDir(self):
print('pushing %s' % self.xpcDir)
# The tests directory can be quite large: 5000 files and growing!
# Sometimes - like on a low-end aws instance running an emulator - the push
# may exceed the default 5 minute timeout, so we increase it here to 10 minutes.
self.initDir(self.remoteScriptsDir)
self.device.push(self.xpcDir, self.remoteScriptsDir, timeout=600)
self.device.chmod(self.remoteScriptsDir, recursive=True, root=True)
def buildTestList(self, test_tags=None, test_paths=None, verify=False):
xpcshell.XPCShellTests.buildTestList(
self, test_tags=test_tags, test_paths=test_paths, verify=verify)
uniqueTestPaths = set([])
for test in self.alltests:
uniqueTestPaths.add(test['here'])
for testdir in uniqueTestPaths:
abbrevTestDir = os.path.relpath(testdir, self.xpcDir)
remoteScriptDir = posixpath.join(self.remoteScriptsDir, abbrevTestDir)
self.pathMapping.append(PathMapping(testdir, remoteScriptDir))
class RemoteReftest(RefTest):
use_marionette = False
resolver_cls = RemoteReftestResolver
def __init__(self, options, scriptDir):
RefTest.__init__(self, options.suite)
self.run_by_manifest = False
self.scriptDir = scriptDir
self.localLogName = options.localLogName
verbose = False
if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug':
verbose = True
print "set verbose!"
self.device = ADBDevice(adb=options.adb_path or 'adb',
device=options.deviceSerial,
test_root=options.remoteTestRoot,
verbose=verbose)
if options.remoteTestRoot is None:
options.remoteTestRoot = posixpath.join(self.device.test_root,
"reftest")
options.remoteProfile = posixpath.join(options.remoteTestRoot,
"profile")
options.remoteLogFile = posixpath.join(options.remoteTestRoot,
"reftest.log")
options.logFile = options.remoteLogFile
self.remoteProfile = options.remoteProfile
self.remoteTestRoot = options.remoteTestRoot
if not options.ignoreWindowSize:
parts = self.device.get_info('screen')['screen'][0].split()
width = int(parts[0].split(':')[1])
height = int(parts[1].split(':')[1])
if (width < 1366 or height < 1050):
self.error("ERROR: Invalid screen resolution %sx%s, "
"please adjust to 1366x1050 or higher" %
(width, height))
self._populate_logger(options)
self.outputHandler = OutputHandler(self.log, options.utilityPath,
options.symbolsPath)
# RemoteAutomation.py's 'messageLogger' is also used by mochitest. Mimic a mochitest
# MessageLogger object to re-use this code path.
self.outputHandler.write = self.outputHandler.__call__
args = {'messageLogger': self.outputHandler}
self.automation = RemoteAutomation(self.device,
appName=options.app,
remoteProfile=self.remoteProfile,
remoteLog=options.remoteLogFile,
processArgs=args)
self.environment = self.automation.environment
if self.automation.IS_DEBUG_BUILD:
self.SERVER_STARTUP_TIMEOUT = 180
else:
self.SERVER_STARTUP_TIMEOUT = 90
self.remoteCache = os.path.join(options.remoteTestRoot, "cache/")
# Check that Firefox is installed
expected = options.app.split('/')[-1]
if not self.device.is_app_installed(expected):
raise Exception("%s is not installed on this device" % expected)
self.device.clear_logcat()
self.device.rm(self.remoteCache, force=True, recursive=True)
procName = options.app.split('/')[-1]
self.device.stop_application(procName)
if self.device.process_exist(procName):
self.log.error("unable to kill %s before starting tests!" %
procName)
def findPath(self, paths, filename=None):
for path in paths:
p = path
if filename:
p = os.path.join(p, filename)
if os.path.exists(self.getFullPath(p)):
return path
return None
def startWebServer(self, options):
""" Create the webserver on the host and start it up """
remoteXrePath = options.xrePath
remoteUtilityPath = options.utilityPath
localAutomation = Automation()
localAutomation.IS_WIN32 = False
localAutomation.IS_LINUX = False
localAutomation.IS_MAC = False
localAutomation.UNIXISH = False
hostos = sys.platform
if (hostos == 'mac' or hostos == 'darwin'):
localAutomation.IS_MAC = True
elif (hostos == 'linux' or hostos == 'linux2'):
localAutomation.IS_LINUX = True
localAutomation.UNIXISH = True
elif (hostos == 'win32' or hostos == 'win64'):
localAutomation.BIN_SUFFIX = ".exe"
localAutomation.IS_WIN32 = True
paths = [options.xrePath, localAutomation.DIST_BIN]
options.xrePath = self.findPath(paths)
if options.xrePath is None:
print(
"ERROR: unable to find xulrunner path for %s, "
"please specify with --xre-path" % (os.name))
return 1
paths.append("bin")
paths.append(os.path.join("..", "bin"))
xpcshell = "xpcshell"
if (os.name == "nt"):
xpcshell += ".exe"
if (options.utilityPath):
paths.insert(0, options.utilityPath)
options.utilityPath = self.findPath(paths, xpcshell)
if options.utilityPath is None:
print(
"ERROR: unable to find utility path for %s, "
"please specify with --utility-path" % (os.name))
return 1
options.serverProfilePath = tempfile.mkdtemp()
self.server = ReftestServer(localAutomation, options, self.scriptDir)
retVal = self.server.start()
if retVal:
return retVal
retVal = self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT)
if retVal:
return retVal
options.xrePath = remoteXrePath
options.utilityPath = remoteUtilityPath
return 0
def stopWebServer(self, options):
self.server.stop()
def killNamedProc(self, pname, orphans=True):
""" Kill processes matching the given command name """
self.log.info("Checking for %s processes..." % pname)
for proc in psutil.process_iter():
try:
if proc.name() == pname:
procd = proc.as_dict(
attrs=['pid', 'ppid', 'name', 'username'])
if proc.ppid() == 1 or not orphans:
self.log.info("killing %s" % procd)
try:
os.kill(proc.pid,
getattr(signal, "SIGKILL", signal.SIGTERM))
except Exception as e:
self.log.info("Failed to kill process %d: %s" %
(proc.pid, str(e)))
else:
self.log.info("NOT killing %s (not an orphan?)" %
procd)
except Exception:
# may not be able to access process info for all processes
continue
def createReftestProfile(self, options, **kwargs):
profile = RefTest.createReftestProfile(self,
options,
server=options.remoteWebServer,
port=options.httpPort,
**kwargs)
profileDir = profile.profile
prefs = {}
prefs["app.update.url.android"] = ""
prefs["reftest.remote"] = True
prefs[
"datareporting.policy.dataSubmissionPolicyBypassAcceptance"] = True
# move necko cache to a location that can be cleaned up
prefs["browser.cache.disk.parent_directory"] = self.remoteCache
prefs["layout.css.devPixelsPerPx"] = "1.0"
# Because Fennec is a little wacky (see bug 1156817) we need to load the
# reftest pages at 1.0 zoom, rather than zooming to fit the CSS viewport.
prefs["apz.allow_zooming"] = False
# Set the extra prefs.
profile.set_preferences(prefs)
try:
self.device.push(profileDir, options.remoteProfile)
self.device.chmod(options.remoteProfile, recursive=True, root=True)
except Exception:
print "Automation Error: Failed to copy profiledir to device"
raise
return profile
def printDeviceInfo(self, printLogcat=False):
try:
if printLogcat:
logcat = self.device.get_logcat(
filter_out_regexps=fennecLogcatFilters)
for l in logcat:
ul = l.decode('utf-8', errors='replace')
sl = ul.encode('iso8859-1', errors='replace')
print "%s\n" % sl
print "Device info:"
devinfo = self.device.get_info()
for category in devinfo:
if type(devinfo[category]) is list:
print " %s:" % category
for item in devinfo[category]:
print " %s" % item
else:
print " %s: %s" % (category, devinfo[category])
print "Test root: %s" % self.device.test_root
except ADBTimeoutError:
raise
except Exception as e:
print "WARNING: Error getting device information: %s" % str(e)
def environment(self, **kwargs):
return self.automation.environment(**kwargs)
def buildBrowserEnv(self, options, profileDir):
browserEnv = RefTest.buildBrowserEnv(self, options, profileDir)
# remove desktop environment not used on device
if "XPCOM_MEM_BLOAT_LOG" in browserEnv:
del browserEnv["XPCOM_MEM_BLOAT_LOG"]
return browserEnv
def runApp(self,
options,
cmdargs=None,
timeout=None,
debuggerInfo=None,
symbolsPath=None,
valgrindPath=None,
valgrindArgs=None,
valgrindSuppFiles=None,
**profileArgs):
if cmdargs is None:
cmdargs = []
if self.use_marionette:
cmdargs.append('-marionette')
binary = options.app
profile = self.createReftestProfile(options, **profileArgs)
# browser environment
env = self.buildBrowserEnv(options, profile.profile)
self.log.info("Running with e10s: {}".format(options.e10s))
status, self.lastTestSeen = self.automation.runApp(
None,
env,
binary,
profile.profile,
cmdargs,
utilityPath=options.utilityPath,
xrePath=options.xrePath,
debuggerInfo=debuggerInfo,
symbolsPath=symbolsPath,
timeout=timeout,
e10s=options.e10s)
self.cleanup(profile.profile)
return status
def cleanup(self, profileDir):
self.device.rm(self.remoteTestRoot, force=True, recursive=True)
self.device.rm(self.remoteProfile, force=True, recursive=True)
self.device.rm(self.remoteCache, force=True, recursive=True)
RefTest.cleanup(self, profileDir)
class RaptorAndroid(Raptor):
def __init__(self, app, binary, run_local=False, obj_path=None,
gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None,
symbols_path=None, host=None, power_test=False, is_release_build=False,
debug_mode=False, activity=None):
Raptor.__init__(self, app, binary, run_local, obj_path, gecko_profile,
gecko_profile_interval, gecko_profile_entries, symbols_path, host,
power_test, is_release_build, debug_mode)
# on android, when creating the browser profile, we want to use a 'firefox' type profile
self.profile_class = "firefox"
self.config['activity'] = activity
def create_browser_handler(self):
# create the android device handler; it gets initiated and sets up adb etc
self.log.info("creating android device handler using mozdevice")
self.device = ADBDevice(verbose=True)
self.device.clear_logcat()
self.clear_app_data()
def clear_app_data(self):
self.log.info("clearing %s app data" % self.config['binary'])
self.device.shell("pm clear %s" % self.config['binary'])
def create_raptor_sdcard_folder(self):
# for android/geckoview, create a top-level raptor folder on the device
# sdcard; if it already exists remove it so we start fresh each time
self.device_raptor_dir = "/sdcard/raptor"
self.config['device_raptor_dir'] = self.device_raptor_dir
if self.device.is_dir(self.device_raptor_dir):
self.log.info("deleting existing device raptor dir: %s" % self.device_raptor_dir)
self.device.rm(self.device_raptor_dir, recursive=True)
self.log.info("creating raptor folder on sdcard: %s" % self.device_raptor_dir)
self.device.mkdir(self.device_raptor_dir)
self.device.chmod(self.device_raptor_dir, recursive=True)
def copy_profile_onto_device(self):
# for geckoview/fennec we must copy the profile onto the device and set perms
if not self.device.is_app_installed(self.config['binary']):
raise Exception('%s is not installed' % self.config['binary'])
self.device_profile = os.path.join(self.device_raptor_dir, "profile")
if self.device.is_dir(self.device_profile):
self.log.info("deleting existing device profile folder: %s" % self.device_profile)
self.device.rm(self.device_profile, recursive=True)
self.log.info("creating profile folder on device: %s" % self.device_profile)
self.device.mkdir(self.device_profile)
self.log.info("copying firefox profile onto the device")
self.log.info("note: the profile folder being copied is: %s" % self.profile.profile)
self.log.info('the adb push cmd copies that profile dir to a new temp dir before copy')
self.device.push(self.profile.profile, self.device_profile)
self.device.chmod(self.device_profile, recursive=True)
def turn_on_android_app_proxy(self):
# for geckoview/android pageload playback we can't use a policy to turn on the
# proxy; we need to set prefs instead; note that the 'host' may be different
# than '127.0.0.1' so we must set the prefs accordingly
self.log.info("setting profile prefs to turn on the android app proxy")
proxy_prefs = {}
proxy_prefs["network.proxy.type"] = 1
proxy_prefs["network.proxy.http"] = self.config['host']
proxy_prefs["network.proxy.http_port"] = 8080
proxy_prefs["network.proxy.ssl"] = self.config['host']
proxy_prefs["network.proxy.ssl_port"] = 8080
proxy_prefs["network.proxy.no_proxies_on"] = self.config['host']
self.profile.set_preferences(proxy_prefs)
def launch_firefox_android_app(self):
self.log.info("starting %s" % self.config['app'])
extra_args = ["-profile", self.device_profile,
"--es", "env0", "LOG_VERBOSE=1",
"--es", "env1", "R_LOG_LEVEL=6"]
try:
# make sure the android app is not already running
self.device.stop_application(self.config['binary'])
if self.config['app'] == "fennec":
self.device.launch_fennec(self.config['binary'],
extra_args=extra_args,
url='about:blank',
fail_if_running=False)
else:
self.device.launch_activity(self.config['binary'],
self.config['activity'],
extra_args=extra_args,
url='about:blank',
e10s=True,
fail_if_running=False)
except Exception as e:
self.log.error("Exception launching %s" % self.config['binary'])
self.log.error("Exception: %s %s" % (type(e).__name__, str(e)))
if self.config['power_test']:
finish_geckoview_power_test(self)
raise
# give our control server the device and app info
self.control_server.device = self.device
self.control_server.app_name = self.config['binary']
def run_test(self, test, timeout=None):
if self.config['power_test']:
init_geckoview_power_test(self)
self.run_test_setup(test)
self.create_raptor_sdcard_folder()
if test.get('playback', None) is not None:
self.start_playback(test)
if self.config['host'] not in ('localhost', '127.0.0.1'):
self.delete_proxy_settings_from_profile()
if test.get('playback', None) is not None:
self.turn_on_android_app_proxy()
self.copy_profile_onto_device()
# now start the browser/app under test
self.launch_firefox_android_app()
# set our control server flag to indicate we are running the browser/app
self.control_server._finished = False
self.wait_for_test_finish(test, timeout)
if self.config['power_test']:
finish_geckoview_power_test(self)
self.run_test_teardown()
# in debug mode, and running locally, leave the browser running
if self.debug_mode and self.config['run_local']:
self.log.info("* debug-mode enabled - please shutdown the browser manually...")
self.runner.wait(timeout=None)
def check_for_crashes(self):
# Turn off verbose to prevent logcat from being inserted into the main log.
verbose = self.device._verbose
self.device._verbose = False
logcat = self.device.get_logcat()
self.device._verbose = verbose
if logcat:
if mozcrash.check_for_java_exception(logcat, "raptor"):
return
try:
dump_dir = tempfile.mkdtemp()
remote_dir = posixpath.join(self.device_profile, 'minidumps')
if not self.device.is_dir(remote_dir):
self.log.error("No crash directory (%s) found on remote device" % remote_dir)
return
self.device.pull(remote_dir, dump_dir)
mozcrash.log_crashes(self.log, dump_dir, self.config['symbols_path'])
finally:
try:
shutil.rmtree(dump_dir)
except Exception:
self.log.warning("unable to remove directory: %s" % dump_dir)
class WebExtensionAndroid(PerftestAndroid, WebExtension):
def __init__(self, app, binary, activity=None, intent=None, **kwargs):
super(WebExtensionAndroid, self).__init__(app,
binary,
profile_class="firefox",
**kwargs)
self.config.update({"activity": activity, "intent": intent})
self.remote_test_root = os.path.abspath(
os.path.join(os.sep, "sdcard", "raptor"))
self.remote_profile = os.path.join(self.remote_test_root, "profile")
self.os_baseline_data = None
self.power_test_time = None
self.screen_off_timeout = 0
self.screen_brightness = 127
self.app_launched = False
def setup_adb_device(self):
if self.device is None:
self.device = ADBDevice(verbose=True)
tune_performance(self.device, log=LOG)
if self.config['power_test']:
disable_charging(self.device)
LOG.info("creating remote root folder for raptor: %s" %
self.remote_test_root)
self.device.rm(self.remote_test_root, force=True, recursive=True)
self.device.mkdir(self.remote_test_root)
self.device.chmod(self.remote_test_root, recursive=True, root=True)
self.clear_app_data()
self.set_debug_app_flag()
def write_android_app_config(self):
# geckoview supports having a local on-device config file; use this file
# to tell the app to use the specified browser profile, as well as other opts
# on-device: /data/local/tmp/com.yourcompany.yourapp-geckoview-config.yaml
# https://mozilla.github.io/geckoview/tutorials/automation.html#configuration-file-format
# only supported for geckoview apps
if self.config["app"] == "fennec":
return
LOG.info("creating android app config.yml")
yml_config_data = dict(
args=[
"--profile",
self.remote_profile,
"use_multiprocess",
self.config["e10s"],
],
env=dict(
LOG_VERBOSE=1,
R_LOG_LEVEL=6,
MOZ_WEBRENDER=int(self.config["enable_webrender"]),
),
)
yml_name = "%s-geckoview-config.yaml" % self.config["binary"]
yml_on_host = os.path.join(tempfile.mkdtemp(), yml_name)
write_yml_file(yml_on_host, yml_config_data)
yml_on_device = os.path.join("/data", "local", "tmp", yml_name)
try:
LOG.info("copying %s to device: %s" % (yml_on_host, yml_on_device))
self.device.rm(yml_on_device, force=True, recursive=True)
self.device.push(yml_on_host, yml_on_device)
except Exception:
LOG.critical("failed to push %s to device!" % yml_on_device)
raise
def log_android_device_temperature(self):
try:
# retrieve and log the android device temperature
thermal_zone0 = self.device.shell_output(
"cat sys/class/thermal/thermal_zone0/temp")
thermal_zone0 = float(thermal_zone0)
zone_type = self.device.shell_output(
"cat sys/class/thermal/thermal_zone0/type")
LOG.info("(thermal_zone0) device temperature: %.3f zone type: %s" %
(thermal_zone0 / 1000, zone_type))
except Exception as exc:
LOG.warning("Unexpected error: {} - {}".format(
exc.__class__.__name__, exc))
def launch_firefox_android_app(self, test_name):
LOG.info("starting %s" % self.config["app"])
extra_args = [
"-profile",
self.remote_profile,
"--es",
"env0",
"LOG_VERBOSE=1",
"--es",
"env1",
"R_LOG_LEVEL=6",
"--es",
"env2",
"MOZ_WEBRENDER=%d" % self.config["enable_webrender"],
]
try:
# make sure the android app is not already running
self.device.stop_application(self.config["binary"])
if self.config["app"] == "fennec":
self.device.launch_fennec(
self.config["binary"],
extra_args=extra_args,
url="about:blank",
fail_if_running=False,
)
else:
# command line 'extra' args not used with geckoview apps; instead we use
# an on-device config.yml file (see write_android_app_config)
self.device.launch_application(
self.config["binary"],
self.config["activity"],
self.config["intent"],
extras=None,
url="about:blank",
fail_if_running=False,
)
# Check if app has started and it's running
if not self.device.process_exist(self.config["binary"]):
raise Exception(
"Error launching %s. App did not start properly!" %
self.config["binary"])
self.app_launched = True
except Exception as e:
LOG.error("Exception launching %s" % self.config["binary"])
LOG.error("Exception: %s %s" % (type(e).__name__, str(e)))
if self.config["power_test"]:
finish_android_power_test(self, test_name)
raise
# give our control server the device and app info
self.control_server.device = self.device
self.control_server.app_name = self.config["binary"]
def copy_cert_db(self, source_dir, target_dir):
# copy browser cert db (that was previously created via certutil) from source to target
cert_db_files = ["pkcs11.txt", "key4.db", "cert9.db"]
for next_file in cert_db_files:
_source = os.path.join(source_dir, next_file)
_dest = os.path.join(target_dir, next_file)
if os.path.exists(_source):
LOG.info("copying %s to %s" % (_source, _dest))
shutil.copyfile(_source, _dest)
else:
LOG.critical("unable to find ssl cert db file: %s" % _source)
def run_tests(self, tests, test_names):
self.setup_adb_device()
return super(WebExtensionAndroid, self).run_tests(tests, test_names)
def run_test_setup(self, test):
super(WebExtensionAndroid, self).run_test_setup(test)
self.set_reverse_ports()
def run_test_teardown(self, test):
LOG.info("removing reverse socket connections")
self.device.remove_socket_connections("reverse")
super(WebExtensionAndroid, self).run_test_teardown(test)
def run_test(self, test, timeout):
# tests will be run warm (i.e. NO browser restart between page-cycles)
# unless otheriwse specified in the test INI by using 'cold = true'
try:
if self.config["power_test"]:
# gather OS baseline data
init_android_power_test(self)
LOG.info("Running OS baseline, pausing for 1 minute...")
time.sleep(60)
LOG.info("Finishing baseline...")
finish_android_power_test(self,
"os-baseline",
os_baseline=True)
# initialize for the test
init_android_power_test(self)
if test.get("cold", False) is True:
self.__run_test_cold(test, timeout)
else:
self.__run_test_warm(test, timeout)
except SignalHandlerException:
self.device.stop_application(self.config["binary"])
if self.config['power_test']:
enable_charging(self.device)
finally:
if self.config["power_test"]:
finish_android_power_test(self, test["name"])
def __run_test_cold(self, test, timeout):
"""
Run the Raptor test but restart the entire browser app between page-cycles.
Note: For page-load tests, playback will only be started once - at the beginning of all
browser cycles, and then stopped after all cycles are finished. The proxy is set via prefs
in the browser profile so those will need to be set again in each new profile/cycle.
Note that instead of using the certutil tool each time to create a db and import the
mitmproxy SSL cert (it's done in mozbase/mozproxy) we will simply copy the existing
cert db from the first cycle's browser profile into the new clean profile; this way
we don't have to re-create the cert db on each browser cycle.
Since we're running in cold-mode, before this point (in manifest.py) the
'expected-browser-cycles' value was already set to the initial 'page-cycles' value;
and the 'page-cycles' value was set to 1 as we want to perform one page-cycle per
browser restart.
The 'browser-cycle' value is the current overall browser start iteration. The control
server will receive the current 'browser-cycle' and the 'expected-browser-cycles' in
each results set received; and will pass that on as part of the results so that the
results processing will know results for multiple browser cycles are being received.
The default will be to run in warm mode; unless 'cold = true' is set in the test INI.
"""
LOG.info(
"test %s is running in cold mode; browser WILL be restarted between "
"page cycles" % test["name"])
for test["browser_cycle"] in range(1, test["expected_browser_cycles"] +
1):
LOG.info("begin browser cycle %d of %d for test %s" %
(test["browser_cycle"], test["expected_browser_cycles"],
test["name"]))
self.run_test_setup(test)
self.clear_app_data()
self.set_debug_app_flag()
if test["browser_cycle"] == 1:
if test.get("playback") is not None:
# an ssl cert db has now been created in the profile; copy it out so we
# can use the same cert db in future test cycles / browser restarts
local_cert_db_dir = tempfile.mkdtemp()
LOG.info(
"backing up browser ssl cert db that was created via certutil"
)
self.copy_cert_db(self.config["local_profile_dir"],
local_cert_db_dir)
if not self.is_localhost:
self.delete_proxy_settings_from_profile()
else:
# double-check to ensure app has been shutdown
self.device.stop_application(self.config["binary"])
# initial browser profile was already created before run_test was called;
# now additional browser cycles we want to create a new one each time
self.build_browser_profile()
if test.get("playback") is not None:
# get cert db from previous cycle profile and copy into new clean profile
# this saves us from having to start playback again / recreate cert db etc.
LOG.info(
"copying existing ssl cert db into new browser profile"
)
self.copy_cert_db(local_cert_db_dir,
self.config["local_profile_dir"])
self.run_test_setup(test)
if test.get("playback") is not None:
self.turn_on_android_app_proxy()
self.copy_profile_to_device()
self.log_android_device_temperature()
# write android app config.yml
self.write_android_app_config()
# now start the browser/app under test
self.launch_firefox_android_app(test["name"])
# set our control server flag to indicate we are running the browser/app
self.control_server._finished = False
if self.config["cpu_test"]:
# start measuring CPU usage
self.cpu_profiler = start_android_cpu_profiler(self)
self.wait_for_test_finish(test, timeout)
# in debug mode, and running locally, leave the browser running
if self.debug_mode and self.config["run_local"]:
LOG.info(
"* debug-mode enabled - please shutdown the browser manually..."
)
self.runner.wait(timeout=None)
# break test execution if a exception is present
if len(self.results_handler.page_timeout_list) > 0:
break
def __run_test_warm(self, test, timeout):
LOG.info(
"test %s is running in warm mode; browser will NOT be restarted between "
"page cycles" % test["name"])
self.run_test_setup(test)
if not self.is_localhost:
self.delete_proxy_settings_from_profile()
if test.get("playback") is not None:
self.turn_on_android_app_proxy()
self.clear_app_data()
self.set_debug_app_flag()
self.copy_profile_to_device()
self.log_android_device_temperature()
# write android app config.yml
self.write_android_app_config()
# now start the browser/app under test
self.launch_firefox_android_app(test["name"])
# set our control server flag to indicate we are running the browser/app
self.control_server._finished = False
if self.config["cpu_test"]:
# start measuring CPU usage
self.cpu_profiler = start_android_cpu_profiler(self)
self.wait_for_test_finish(test, timeout)
# in debug mode, and running locally, leave the browser running
if self.debug_mode and self.config["run_local"]:
LOG.info(
"* debug-mode enabled - please shutdown the browser manually..."
)
def check_for_crashes(self):
super(WebExtensionAndroid, self).check_for_crashes()
if not self.app_launched:
LOG.info(
"skipping check_for_crashes: application has not been launched"
)
return
self.app_launched = False
try:
dump_dir = tempfile.mkdtemp()
remote_dir = posixpath.join(self.remote_profile, "minidumps")
if not self.device.is_dir(remote_dir):
return
self.device.pull(remote_dir, dump_dir)
self.crashes += mozcrash.log_crashes(LOG, dump_dir,
self.config["symbols_path"])
finally:
try:
shutil.rmtree(dump_dir)
except Exception:
LOG.warning("unable to remove directory: %s" % dump_dir)
def clean_up(self):
LOG.info("removing test folder for raptor: %s" % self.remote_test_root)
self.device.rm(self.remote_test_root, force=True, recursive=True)
if self.config['power_test']:
enable_charging(self.device)
super(WebExtensionAndroid, self).clean_up()
class XPCShellRemote(xpcshell.XPCShellTests, object):
def __init__(self, options, log):
xpcshell.XPCShellTests.__init__(self, log)
self.options = options
verbose = False
if options['log_tbpl_level'] == 'debug' or options['log_mach_level'] == 'debug':
verbose = True
self.device = ADBDevice(adb=options['adbPath'] or 'adb',
device=options['deviceSerial'],
test_root=options['remoteTestRoot'],
verbose=verbose)
self.remoteTestRoot = posixpath.join(self.device.test_root, "xpc")
# Add Android version (SDK level) to mozinfo so that manifest entries
# can be conditional on android_version.
mozinfo.info['android_version'] = str(self.device.version)
mozinfo.info['is_emulator'] = self.device._device_serial.startswith('emulator-')
self.localBin = options['localBin']
self.pathMapping = []
# remoteBinDir contains xpcshell and its wrapper script, both of which must
# be executable. Since +x permissions cannot usually be set on /mnt/sdcard,
# and the test root may be on /mnt/sdcard, remoteBinDir is set to be on
# /data/local, always.
self.remoteBinDir = posixpath.join("/data", "local", "xpcb")
# Terse directory names are used here ("c" for the components directory)
# to minimize the length of the command line used to execute
# xpcshell on the remote device. adb has a limit to the number
# of characters used in a shell command, and the xpcshell command
# line can be quite complex.
self.remoteTmpDir = posixpath.join(self.remoteTestRoot, "tmp")
self.remoteScriptsDir = self.remoteTestRoot
self.remoteComponentsDir = posixpath.join(self.remoteTestRoot, "c")
self.remoteModulesDir = posixpath.join(self.remoteTestRoot, "m")
self.remoteMinidumpDir = posixpath.join(self.remoteTestRoot, "minidumps")
self.profileDir = posixpath.join(self.remoteTestRoot, "p")
self.remoteDebugger = options['debugger']
self.remoteDebuggerArgs = options['debuggerArgs']
self.testingModulesDir = options['testingModulesDir']
self.env = {}
if options['objdir']:
self.xpcDir = os.path.join(options['objdir'], "_tests/xpcshell")
elif os.path.isdir(os.path.join(here, 'tests')):
self.xpcDir = os.path.join(here, 'tests')
else:
print("Couldn't find local xpcshell test directory", file=sys.stderr)
sys.exit(1)
self.remoteAPK = None
if options['localAPK']:
self.localAPKContents = ZipFile(options['localAPK'])
self.remoteAPK = posixpath.join(self.remoteBinDir,
os.path.basename(options['localAPK']))
else:
self.localAPKContents = None
if options['setup']:
self.setupTestDir()
self.setupUtilities()
self.setupModules()
self.initDir(self.remoteMinidumpDir)
# data that needs to be passed to the RemoteXPCShellTestThread
self.mobileArgs = {
'device': self.device,
'remoteBinDir': self.remoteBinDir,
'remoteScriptsDir': self.remoteScriptsDir,
'remoteComponentsDir': self.remoteComponentsDir,
'remoteModulesDir': self.remoteModulesDir,
'options': self.options,
'remoteDebugger': self.remoteDebugger,
'pathMapping': self.pathMapping,
'profileDir': self.profileDir,
'remoteTmpDir': self.remoteTmpDir,
'remoteMinidumpDir': self.remoteMinidumpDir,
}
if self.remoteAPK:
self.mobileArgs['remoteAPK'] = self.remoteAPK
def initDir(self, path, mask="777", timeout=None, root=True):
"""Initialize a directory by removing it if it exists, creating it
and changing the permissions."""
self.device.rm(path, recursive=True, force=True, timeout=timeout, root=root)
self.device.mkdir(path, parents=True, timeout=timeout, root=root)
self.device.chmod(path, recursive=True, mask=mask, timeout=timeout, root=root)
def setLD_LIBRARY_PATH(self):
self.env["LD_LIBRARY_PATH"] = self.remoteBinDir
def pushWrapper(self):
# Rather than executing xpcshell directly, this wrapper script is
# used. By setting environment variables and the cwd in the script,
# the length of the per-test command line is shortened. This is
# often important when using ADB, as there is a limit to the length
# of the ADB command line.
localWrapper = tempfile.mktemp()
f = open(localWrapper, "w")
f.write("#!/system/bin/sh\n")
for envkey, envval in self.env.iteritems():
f.write("export %s=%s\n" % (envkey, envval))
f.writelines([
"cd $1\n",
"echo xpcw: cd $1\n",
"shift\n",
"echo xpcw: xpcshell \"$@\"\n",
"%s/xpcshell \"$@\"\n" % self.remoteBinDir])
f.close()
remoteWrapper = posixpath.join(self.remoteBinDir, "xpcw")
self.device.push(localWrapper, remoteWrapper)
self.device.chmod(remoteWrapper, root=True)
os.remove(localWrapper)
def buildPrefsFile(self, extraPrefs):
prefs = super(XPCShellRemote, self).buildPrefsFile(extraPrefs)
remotePrefsFile = posixpath.join(self.remoteTestRoot, 'user.js')
self.device.push(self.prefsFile, remotePrefsFile)
self.device.chmod(remotePrefsFile, root=True)
os.remove(self.prefsFile)
self.prefsFile = remotePrefsFile
return prefs
def buildEnvironment(self):
self.buildCoreEnvironment()
self.setLD_LIBRARY_PATH()
self.env["MOZ_LINKER_CACHE"] = self.remoteBinDir
self.env["GRE_HOME"] = self.remoteBinDir
self.env["XPCSHELL_TEST_PROFILE_DIR"] = self.profileDir
self.env["TMPDIR"] = self.remoteTmpDir
self.env["HOME"] = self.profileDir
self.env["XPCSHELL_TEST_TEMP_DIR"] = self.remoteTmpDir
self.env["XPCSHELL_MINIDUMP_DIR"] = self.remoteMinidumpDir
self.env["MOZ_ANDROID_DATA_DIR"] = self.remoteBinDir
# Guard against intermittent failures to retrieve abi property;
# without an abi, xpcshell cannot find greprefs.js and crashes.
abilistprop = None
abi = None
retries = 0
while not abi and retries < 3:
abi = self.device.get_prop("ro.product.cpu.abi")
retries += 1
if not abi:
raise Exception("failed to get ro.product.cpu.abi from device")
self.log.info("ro.product.cpu.abi %s" % abi)
if self.localAPKContents:
abilist = [abi]
retries = 0
while not abilistprop and retries < 3:
abilistprop = self.device.get_prop("ro.product.cpu.abilist")
retries += 1
self.log.info('ro.product.cpu.abilist %s' % abilistprop)
abi_found = False
names = [n for n in self.localAPKContents.namelist() if n.startswith('lib/')]
self.log.debug("apk names: %s" % names)
if abilistprop and len(abilistprop) > 0:
abilist.extend(abilistprop.split(','))
for abicand in abilist:
abi_found = len([n for n in names if n.startswith('lib/%s' % abicand)]) > 0
if abi_found:
abi = abicand
break
if not abi_found:
self.log.info("failed to get matching abi from apk.")
if len(names) > 0:
self.log.info("device cpu abi not found in apk. Using abi from apk.")
abi = names[0].split('/')[1]
self.log.info("Using abi %s." % abi)
self.env["MOZ_ANDROID_CPU_ABI"] = abi
def setupUtilities(self):
self.initDir(self.remoteTmpDir)
self.initDir(self.remoteBinDir)
remotePrefDir = posixpath.join(self.remoteBinDir, "defaults", "pref")
self.initDir(posixpath.join(remotePrefDir, "extra"))
self.initDir(self.remoteComponentsDir)
local = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'head.js')
remoteFile = posixpath.join(self.remoteScriptsDir, "head.js")
self.device.push(local, remoteFile)
self.device.chmod(remoteFile, root=True)
# The xpcshell binary is required for all tests. Additional binaries
# are required for some tests. This list should be similar to
# TEST_HARNESS_BINS in testing/mochitest/Makefile.in.
binaries = ["xpcshell",
"ssltunnel",
"certutil",
"pk12util",
"BadCertAndPinningServer",
"DelegatedCredentialsServer",
"OCSPStaplingServer",
"GenerateOCSPResponse",
"SanctionsTestServer"]
for fname in binaries:
local = os.path.join(self.localBin, fname)
if os.path.isfile(local):
print("Pushing %s.." % fname, file=sys.stderr)
remoteFile = posixpath.join(self.remoteBinDir, fname)
self.device.push(local, remoteFile)
self.device.chmod(remoteFile, root=True)
else:
print("*** Expected binary %s not found in %s!" %
(fname, self.localBin), file=sys.stderr)
local = os.path.join(self.localBin, "components/httpd.js")
remoteFile = posixpath.join(self.remoteComponentsDir, "httpd.js")
self.device.push(local, remoteFile)
self.device.chmod(remoteFile, root=True)
local = os.path.join(self.localBin, "components/httpd.manifest")
remoteFile = posixpath.join(self.remoteComponentsDir, "httpd.manifest")
self.device.push(local, remoteFile)
self.device.chmod(remoteFile, root=True)
if self.options['localAPK']:
remoteFile = posixpath.join(self.remoteBinDir,
os.path.basename(self.options['localAPK']))
self.device.push(self.options['localAPK'], remoteFile)
self.device.chmod(remoteFile, root=True)
self.pushLibs()
else:
localB2G = os.path.join(self.options['objdir'], "dist", "b2g")
if os.path.exists(localB2G):
self.device.push(localB2G, self.remoteBinDir)
self.device.chmod(self.remoteBinDir, root=True)
else:
raise Exception("unable to install gre: no APK and not b2g")
def pushLibs(self):
elfhack = os.path.join(self.localBin, 'elfhack')
if not os.path.exists(elfhack):
elfhack = None
pushed_libs_count = 0
try:
dir = tempfile.mkdtemp()
for info in self.localAPKContents.infolist():
if info.filename.endswith(".so"):
print("Pushing %s.." % info.filename, file=sys.stderr)
remoteFile = posixpath.join(self.remoteBinDir,
os.path.basename(info.filename))
self.localAPKContents.extract(info, dir)
localFile = os.path.join(dir, info.filename)
with open(localFile) as f:
# Decompress xz-compressed file.
if f.read(5)[1:] == '7zXZ':
cmd = ['xz', '-df', '--suffix', '.so', localFile]
subprocess.check_output(cmd)
# xz strips the ".so" file suffix.
os.rename(localFile[:-3], localFile)
# elfhack -r should provide better crash reports
if elfhack:
cmd = [elfhack, '-r', localFile]
subprocess.check_output(cmd)
self.device.push(localFile, remoteFile)
pushed_libs_count += 1
self.device.chmod(remoteFile, root=True)
finally:
shutil.rmtree(dir)
return pushed_libs_count
def setupModules(self):
if self.testingModulesDir:
self.device.push(self.testingModulesDir, self.remoteModulesDir)
self.device.chmod(self.remoteModulesDir, root=True)
def setupTestDir(self):
print('pushing %s' % self.xpcDir)
# The tests directory can be quite large: 5000 files and growing!
# Sometimes - like on a low-end aws instance running an emulator - the push
# may exceed the default 5 minute timeout, so we increase it here to 10 minutes.
self.device.rm(self.remoteScriptsDir, recursive=True, force=True, timeout=None, root=True)
self.device.push(self.xpcDir, self.remoteScriptsDir, timeout=600)
self.device.chmod(self.remoteScriptsDir, recursive=True, root=True)
def setupSocketConnections(self):
# make node host ports visible to device
if "MOZHTTP2_PORT" in self.env:
port = "tcp:{}".format(self.env["MOZHTTP2_PORT"])
self.device.create_socket_connection(ADBDevice.SOCKET_DIRECTON_REVERSE, port, port)
self.log.info("reversed MOZHTTP2_PORT connection for port " + port)
if "MOZNODE_EXEC_PORT" in self.env:
port = "tcp:{}".format(self.env["MOZNODE_EXEC_PORT"])
self.device.create_socket_connection(ADBDevice.SOCKET_DIRECTON_REVERSE, port, port)
self.log.info("reversed MOZNODE_EXEC_PORT connection for port " + port)
def buildTestList(self, test_tags=None, test_paths=None, verify=False):
xpcshell.XPCShellTests.buildTestList(
self, test_tags=test_tags, test_paths=test_paths, verify=verify)
uniqueTestPaths = set([])
for test in self.alltests:
uniqueTestPaths.add(test['here'])
for testdir in uniqueTestPaths:
abbrevTestDir = os.path.relpath(testdir, self.xpcDir)
remoteScriptDir = posixpath.join(self.remoteScriptsDir, abbrevTestDir)
self.pathMapping.append(PathMapping(testdir, remoteScriptDir))
# This is not related to building the test list, but since this is called late
# in the test suite run, this is a convenient place to finalize preparations;
# in particular, these operations cannot be executed much earlier because
# self.env may not be finalized.
self.setupSocketConnections()
if self.options['setup']:
self.pushWrapper()
class RaptorAndroid(Raptor):
def __init__(self,
app,
binary,
run_local=False,
obj_path=None,
gecko_profile=False,
gecko_profile_interval=None,
gecko_profile_entries=None,
symbols_path=None,
host=None,
power_test=False,
is_release_build=False,
debug_mode=False,
activity=None):
Raptor.__init__(self, app, binary, run_local, obj_path, gecko_profile,
gecko_profile_interval, gecko_profile_entries,
symbols_path, host, power_test, is_release_build,
debug_mode)
# on android, when creating the browser profile, we want to use a 'firefox' type profile
self.profile_class = "firefox"
self.config['activity'] = activity
def create_browser_handler(self):
# create the android device handler; it gets initiated and sets up adb etc
self.log.info("creating android device handler using mozdevice")
self.device = ADBDevice(verbose=True)
self.device.clear_logcat()
self.clear_app_data()
def clear_app_data(self):
self.log.info("clearing %s app data" % self.config['binary'])
self.device.shell("pm clear %s" % self.config['binary'])
def create_raptor_sdcard_folder(self):
# for android/geckoview, create a top-level raptor folder on the device
# sdcard; if it already exists remove it so we start fresh each time
self.device_raptor_dir = "/sdcard/raptor"
self.config['device_raptor_dir'] = self.device_raptor_dir
if self.device.is_dir(self.device_raptor_dir):
self.log.info("deleting existing device raptor dir: %s" %
self.device_raptor_dir)
self.device.rm(self.device_raptor_dir, recursive=True)
self.log.info("creating raptor folder on sdcard: %s" %
self.device_raptor_dir)
self.device.mkdir(self.device_raptor_dir)
self.device.chmod(self.device_raptor_dir, recursive=True)
def copy_profile_onto_device(self):
# for geckoview/fennec we must copy the profile onto the device and set perms
if not self.device.is_app_installed(self.config['binary']):
raise Exception('%s is not installed' % self.config['binary'])
self.device_profile = os.path.join(self.device_raptor_dir, "profile")
if self.device.is_dir(self.device_profile):
self.log.info("deleting existing device profile folder: %s" %
self.device_profile)
self.device.rm(self.device_profile, recursive=True)
self.log.info("creating profile folder on device: %s" %
self.device_profile)
self.device.mkdir(self.device_profile)
self.log.info("copying firefox profile onto the device")
self.log.info("note: the profile folder being copied is: %s" %
self.profile.profile)
self.log.info(
'the adb push cmd copies that profile dir to a new temp dir before copy'
)
self.device.push(self.profile.profile, self.device_profile)
self.device.chmod(self.device_profile, recursive=True)
def turn_on_android_app_proxy(self):
# for geckoview/android pageload playback we can't use a policy to turn on the
# proxy; we need to set prefs instead; note that the 'host' may be different
# than '127.0.0.1' so we must set the prefs accordingly
self.log.info("setting profile prefs to turn on the android app proxy")
proxy_prefs = {}
proxy_prefs["network.proxy.type"] = 1
proxy_prefs["network.proxy.http"] = self.config['host']
proxy_prefs["network.proxy.http_port"] = 8080
proxy_prefs["network.proxy.ssl"] = self.config['host']
proxy_prefs["network.proxy.ssl_port"] = 8080
proxy_prefs["network.proxy.no_proxies_on"] = self.config['host']
self.profile.set_preferences(proxy_prefs)
def launch_firefox_android_app(self):
self.log.info("starting %s" % self.config['app'])
extra_args = [
"-profile", self.device_profile, "--es", "env0", "LOG_VERBOSE=1",
"--es", "env1", "R_LOG_LEVEL=6"
]
try:
# make sure the android app is not already running
self.device.stop_application(self.config['binary'])
if self.config['app'] == "fennec":
self.device.launch_fennec(self.config['binary'],
extra_args=extra_args,
url='about:blank',
fail_if_running=False)
else:
self.device.launch_activity(self.config['binary'],
self.config['activity'],
extra_args=extra_args,
url='about:blank',
e10s=True,
fail_if_running=False)
except Exception as e:
self.log.error("Exception launching %s" % self.config['binary'])
self.log.error("Exception: %s %s" % (type(e).__name__, str(e)))
if self.config['power_test']:
finish_geckoview_power_test(self)
raise
# give our control server the device and app info
self.control_server.device = self.device
self.control_server.app_name = self.config['binary']
def copy_cert_db(self, source_dir, target_dir):
# copy browser cert db (that was previously created via certutil) from source to target
cert_db_files = ['pkcs11.txt', 'key4.db', 'cert9.db']
for next_file in cert_db_files:
_source = os.path.join(source_dir, next_file)
_dest = os.path.join(target_dir, next_file)
if os.path.exists(_source):
self.log.info("copying %s to %s" % (_source, _dest))
shutil.copyfile(_source, _dest)
else:
self.log.critical("unable to find ssl cert db file: %s" %
_source)
def run_test(self, test, timeout=None):
# tests will be run warm (i.e. NO browser restart between page-cycles)
# unless otheriwse specified in the test INI by using 'cold = true'
if test.get('cold', False) is True:
self.run_test_cold(test, timeout)
else:
self.run_test_warm(test, timeout)
def run_test_cold(self, test, timeout=None):
'''
Run the Raptor test but restart the entire browser app between page-cycles.
Note: For page-load tests, playback will only be started once - at the beginning of all
browser cycles, and then stopped after all cycles are finished. The proxy is set via prefs
in the browser profile so those will need to be set again in each new profile/cycle.
Note that instead of using the certutil tool each time to create a db and import the
mitmproxy SSL cert (it's done in mozbase/mozproxy) we will simply copy the existing
cert db from the first cycle's browser profile into the new clean profile; this way
we don't have to re-create the cert db on each browser cycle.
Since we're running in cold-mode, before this point (in manifest.py) the
'expected-browser-cycles' value was already set to the initial 'page-cycles' value;
and the 'page-cycles' value was set to 1 as we want to perform one page-cycle per
browser restart.
The 'browser-cycle' value is the current overall browser start iteration. The control
server will receive the current 'browser-cycle' and the 'expected-browser-cycles' in
each results set received; and will pass that on as part of the results so that the
results processing will know results for multiple browser cycles are being received.
The default will be to run in warm mode; unless 'cold = true' is set in the test INI.
'''
self.log.info(
"test %s is running in cold mode; browser WILL be restarted between "
"page cycles" % test['name'])
if self.config['power_test']:
init_geckoview_power_test(self)
for test['browser_cycle'] in range(1, test['expected_browser_cycles'] +
1):
self.log.info("begin browser cycle %d of %d for test %s" %
(test['browser_cycle'],
test['expected_browser_cycles'], test['name']))
self.run_test_setup(test)
if test['browser_cycle'] == 1:
self.create_raptor_sdcard_folder()
if test.get('playback', None) is not None:
self.start_playback(test)
# an ssl cert db has now been created in the profile; copy it out so we
# can use the same cert db in future test cycles / browser restarts
local_cert_db_dir = tempfile.mkdtemp()
self.log.info(
"backing up browser ssl cert db that was created via certutil"
)
self.copy_cert_db(self.config['local_profile_dir'],
local_cert_db_dir)
if self.config['host'] not in ('localhost', '127.0.0.1'):
self.delete_proxy_settings_from_profile()
else:
# double-check to ensure app has been shutdown
self.device.stop_application(self.config['binary'])
# clear the android app data before the next app startup
self.clear_app_data()
# initial browser profile was already created before run_test was called;
# now additional browser cycles we want to create a new one each time
self.create_browser_profile()
# get cert db from previous cycle profile and copy into new clean profile
# this saves us from having to start playback again / recreate cert db etc.
self.log.info(
"copying existing ssl cert db into new browser profile")
self.copy_cert_db(local_cert_db_dir,
self.config['local_profile_dir'])
self.run_test_setup(test)
if test.get('playback', None) is not None:
self.turn_on_android_app_proxy()
self.copy_profile_onto_device()
# now start the browser/app under test
self.launch_firefox_android_app()
# set our control server flag to indicate we are running the browser/app
self.control_server._finished = False
self.wait_for_test_finish(test, timeout)
# in debug mode, and running locally, leave the browser running
if self.debug_mode and self.config['run_local']:
self.log.info(
"* debug-mode enabled - please shutdown the browser manually..."
)
self.runner.wait(timeout=None)
if self.config['power_test']:
finish_geckoview_power_test(self)
self.run_test_teardown()
def run_test_warm(self, test, timeout=None):
self.log.info(
"test %s is running in warm mode; browser will NOT be restarted between "
"page cycles" % test['name'])
if self.config['power_test']:
init_geckoview_power_test(self)
self.run_test_setup(test)
self.create_raptor_sdcard_folder()
if test.get('playback', None) is not None:
self.start_playback(test)
if self.config['host'] not in ('localhost', '127.0.0.1'):
self.delete_proxy_settings_from_profile()
if test.get('playback', None) is not None:
self.turn_on_android_app_proxy()
self.copy_profile_onto_device()
# now start the browser/app under test
self.launch_firefox_android_app()
# set our control server flag to indicate we are running the browser/app
self.control_server._finished = False
self.wait_for_test_finish(test, timeout)
if self.config['power_test']:
finish_geckoview_power_test(self)
self.run_test_teardown()
# in debug mode, and running locally, leave the browser running
if self.debug_mode and self.config['run_local']:
self.log.info(
"* debug-mode enabled - please shutdown the browser manually..."
)
self.runner.wait(timeout=None)
def check_for_crashes(self):
# Turn off verbose to prevent logcat from being inserted into the main log.
verbose = self.device._verbose
self.device._verbose = False
logcat = self.device.get_logcat()
self.device._verbose = verbose
if logcat:
if mozcrash.check_for_java_exception(logcat, "raptor"):
return
try:
dump_dir = tempfile.mkdtemp()
remote_dir = posixpath.join(self.device_profile, 'minidumps')
if not self.device.is_dir(remote_dir):
self.log.error(
"No crash directory (%s) found on remote device" %
remote_dir)
return
self.device.pull(remote_dir, dump_dir)
mozcrash.log_crashes(self.log, dump_dir,
self.config['symbols_path'])
finally:
try:
shutil.rmtree(dump_dir)
except Exception:
self.log.warning("unable to remove directory: %s" % dump_dir)
class RobocopTestRunner(MochitestDesktop):
"""
A test harness for Robocop. Robocop tests are UI tests for Firefox for Android,
based on the Robotium test framework. This harness leverages some functionality
from mochitest, for convenience.
"""
# Some robocop tests run for >60 seconds without generating any output.
NO_OUTPUT_TIMEOUT = 180
def __init__(self, options, message_logger):
"""
Simple one-time initialization.
"""
MochitestDesktop.__init__(self, options.flavor, vars(options))
verbose = False
if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug':
verbose = True
self.device = ADBDevice(adb=options.adbPath or 'adb',
device=options.deviceSerial,
test_root=options.remoteTestRoot,
verbose=verbose)
# Check that Firefox is installed
expected = options.app.split('/')[-1]
if not self.device.is_app_installed(expected):
raise Exception("%s is not installed on this device" % expected)
options.logFile = "robocop.log"
if options.remoteTestRoot is None:
options.remoteTestRoot = self.device.test_root
self.remoteProfile = posixpath.join(options.remoteTestRoot, "profile")
self.remoteProfileCopy = posixpath.join(options.remoteTestRoot,
"profile-copy")
self.remoteModulesDir = posixpath.join(options.remoteTestRoot,
"modules/")
self.remoteConfigFile = posixpath.join(options.remoteTestRoot,
"robotium.config")
self.remoteLogFile = posixpath.join(options.remoteTestRoot, "logs",
"robocop.log")
self.options = options
process_args = {'messageLogger': message_logger}
self.auto = RemoteAutomation(self.device,
options.remoteappname,
self.remoteProfile,
self.remoteLogFile,
processArgs=process_args)
self.environment = self.auto.environment
self.remoteScreenshots = "/mnt/sdcard/Robotium-Screenshots"
self.remoteMozLog = posixpath.join(options.remoteTestRoot, "mozlog")
self.localLog = options.logFile
self.localProfile = None
self.certdbNew = True
self.passed = 0
self.failed = 0
self.todo = 0
def startup(self):
"""
Second-stage initialization: One-time initialization which may require cleanup.
"""
# Despite our efforts to clean up servers started by this script, in practice
# we still see infrequent cases where a process is orphaned and interferes
# with future tests, typically because the old server is keeping the port in use.
# Try to avoid those failures by checking for and killing servers before
# trying to start new ones.
self.killNamedProc('ssltunnel')
self.killNamedProc('xpcshell')
self.auto.deleteANRs()
self.auto.deleteTombstones()
procName = self.options.app.split('/')[-1]
self.device.stop_application(procName)
if self.device.process_exist(procName):
self.log.warning("unable to kill %s before running tests!" %
procName)
self.device.rm(self.remoteScreenshots, force=True, recursive=True)
self.device.rm(self.remoteMozLog, force=True, recursive=True)
self.device.mkdir(self.remoteMozLog)
logParent = posixpath.dirname(self.remoteLogFile)
self.device.rm(logParent, force=True, recursive=True)
self.device.mkdir(logParent)
# Add Android version (SDK level) to mozinfo so that manifest entries
# can be conditional on android_version.
self.log.info(
"Android sdk version '%s'; will use this to filter manifests" %
str(self.device.version))
mozinfo.info['android_version'] = str(self.device.version)
if self.options.robocopApk:
self.device.install_app(self.options.robocopApk, replace=True)
self.log.debug("Robocop APK %s installed" %
self.options.robocopApk)
# Display remote diagnostics; if running in mach, keep output terse.
if self.options.log_mach is None:
self.printDeviceInfo()
self.setupLocalPaths()
self.buildProfile()
self.startServers(self.options, debuggerInfo=None)
self.log.debug("Servers started")
def cleanup(self):
"""
Cleanup at end of job run.
"""
self.log.debug("Cleaning up...")
self.stopServers()
self.device.stop_application(self.options.app.split('/')[-1])
uploadDir = os.environ.get('MOZ_UPLOAD_DIR', None)
if uploadDir:
self.log.debug(
"Pulling any remote moz logs and screenshots to %s." %
uploadDir)
if self.device.is_dir(self.remoteMozLog):
self.device.pull(self.remoteMozLog, uploadDir)
if self.device.is_dir(self.remoteScreenshots):
self.device.pull(self.remoteScreenshots, uploadDir)
MochitestDesktop.cleanup(self, self.options)
if self.localProfile:
mozfile.remove(self.localProfile)
self.device.rm(self.remoteProfile, force=True, recursive=True)
self.device.rm(self.remoteProfileCopy, force=True, recursive=True)
self.device.rm(self.remoteScreenshots, force=True, recursive=True)
self.device.rm(self.remoteMozLog, force=True, recursive=True)
self.device.rm(self.remoteConfigFile, force=True)
self.device.rm(self.remoteLogFile, force=True)
self.log.debug("Cleanup complete.")
def findPath(self, paths, filename=None):
for path in paths:
p = path
if filename:
p = os.path.join(p, filename)
if os.path.exists(self.getFullPath(p)):
return path
return None
def makeLocalAutomation(self):
localAutomation = Automation()
localAutomation.IS_WIN32 = False
localAutomation.IS_LINUX = False
localAutomation.IS_MAC = False
localAutomation.UNIXISH = False
hostos = sys.platform
if (hostos == 'mac' or hostos == 'darwin'):
localAutomation.IS_MAC = True
elif (hostos == 'linux' or hostos == 'linux2'):
localAutomation.IS_LINUX = True
localAutomation.UNIXISH = True
elif (hostos == 'win32' or hostos == 'win64'):
localAutomation.BIN_SUFFIX = ".exe"
localAutomation.IS_WIN32 = True
return localAutomation
def setupLocalPaths(self):
"""
Setup xrePath and utilityPath and verify xpcshell.
This is similar to switchToLocalPaths in runtestsremote.py.
"""
localAutomation = self.makeLocalAutomation()
paths = [self.options.xrePath, localAutomation.DIST_BIN]
self.options.xrePath = self.findPath(paths)
if self.options.xrePath is None:
self.log.error(
"unable to find xulrunner path for %s, please specify with --xre-path"
% os.name)
sys.exit(1)
self.log.debug("using xre path %s" % self.options.xrePath)
xpcshell = "xpcshell"
if (os.name == "nt"):
xpcshell += ".exe"
if self.options.utilityPath:
paths = [self.options.utilityPath, self.options.xrePath]
else:
paths = [self.options.xrePath]
self.options.utilityPath = self.findPath(paths, xpcshell)
if self.options.utilityPath is None:
self.log.error(
"unable to find utility path for %s, please specify with --utility-path"
% os.name)
sys.exit(1)
self.log.debug("using utility path %s" % self.options.utilityPath)
xpcshell_path = os.path.join(self.options.utilityPath, xpcshell)
if localAutomation.elf_arm(xpcshell_path):
self.log.error('xpcshell at %s is an ARM binary; please use '
'the --utility-path argument to specify the path '
'to a desktop version.' % xpcshell_path)
sys.exit(1)
self.log.debug("xpcshell found at %s" % xpcshell_path)
def buildProfile(self):
"""
Build a profile locally, keep it locally for use by servers and
push a copy to the remote profile-copy directory.
This is similar to buildProfile in runtestsremote.py.
"""
self.options.extraPrefs.append('browser.search.suggest.enabled=true')
self.options.extraPrefs.append('browser.search.suggest.prompted=true')
self.options.extraPrefs.append('layout.css.devPixelsPerPx=1.0')
self.options.extraPrefs.append('browser.chrome.dynamictoolbar=false')
self.options.extraPrefs.append('browser.snippets.enabled=false')
self.options.extraPrefs.append('extensions.autoupdate.enabled=false')
# Override the telemetry init delay for integration testing.
self.options.extraPrefs.append('toolkit.telemetry.initDelay=1')
self.options.extensionsToExclude.extend([
'*****@*****.**',
])
self.extraPrefs = parse_preferences(self.options.extraPrefs)
if self.options.testingModulesDir:
try:
self.device.push(self.options.testingModulesDir,
self.remoteModulesDir)
self.device.chmod(self.remoteModulesDir,
recursive=True,
root=True)
except Exception:
self.log.error(
"Automation Error: Unable to copy test modules to device.")
raise
savedTestingModulesDir = self.options.testingModulesDir
self.options.testingModulesDir = self.remoteModulesDir
else:
savedTestingModulesDir = None
manifest = MochitestDesktop.buildProfile(self, self.options)
if savedTestingModulesDir:
self.options.testingModulesDir = savedTestingModulesDir
self.localProfile = self.options.profilePath
self.log.debug("Profile created at %s" % self.localProfile)
# some files are not needed for robocop; save time by not pushing
os.remove(os.path.join(self.localProfile, 'userChrome.css'))
try:
self.device.push(self.localProfile, self.remoteProfileCopy)
except Exception:
self.log.error(
"Automation Error: Unable to copy profile to device.")
raise
return manifest
def setupRemoteProfile(self):
"""
Remove any remote profile and re-create it.
"""
self.log.debug("Updating remote profile at %s" % self.remoteProfile)
self.device.rm(self.remoteProfile, force=True, recursive=True)
self.device.cp(self.remoteProfileCopy,
self.remoteProfile,
recursive=True)
def parseLocalLog(self):
"""
Read and parse the local log file, noting any failures.
"""
with open(self.localLog) as currentLog:
data = currentLog.readlines()
os.unlink(self.localLog)
start_found = False
end_found = False
fail_found = False
for line in data:
try:
message = json.loads(line)
if not isinstance(message, dict) or 'action' not in message:
continue
except ValueError:
continue
if message['action'] == 'test_end':
end_found = True
start_found = False
break
if start_found and not end_found:
if 'status' in message:
if 'expected' in message:
self.failed += 1
elif message['status'] == 'PASS':
self.passed += 1
elif message['status'] == 'FAIL':
self.todo += 1
if message['action'] == 'test_start':
start_found = True
if 'expected' in message:
fail_found = True
result = 0
if fail_found:
result = 1
if not end_found:
self.log.info(
"PROCESS-CRASH | Automation Error: Missing end of test marker (process crashed?)"
)
result = 1
return result
def logTestSummary(self):
"""
Print a summary of all tests run to stdout, for treeherder parsing
(logging via self.log does not work here).
"""
print("0 INFO TEST-START | Shutdown")
print("1 INFO Passed: %s" % (self.passed))
print("2 INFO Failed: %s" % (self.failed))
print("3 INFO Todo: %s" % (self.todo))
print("4 INFO SimpleTest FINISHED")
if self.failed > 0:
return 1
return 0
def printDeviceInfo(self, printLogcat=False):
"""
Log remote device information and logcat (if requested).
This is similar to printDeviceInfo in runtestsremote.py
"""
try:
if printLogcat:
logcat = self.device.get_logcat(
filter_out_regexps=fennecLogcatFilters)
for l in logcat:
self.log.info(l.decode('utf-8', 'replace'))
self.log.info("Device info:")
devinfo = self.device.get_info()
for category in devinfo:
if type(devinfo[category]) is list:
self.log.info(" %s:" % category)
for item in devinfo[category]:
self.log.info(" %s" % item)
else:
self.log.info(" %s: %s" % (category, devinfo[category]))
self.log.info("Test root: %s" % self.device.test_root)
except ADBTimeoutError:
raise
except Exception as e:
self.log.warning("Error getting device information: %s" % str(e))
def setupRobotiumConfig(self, browserEnv):
"""
Create robotium.config and push it to the device.
"""
fHandle = tempfile.NamedTemporaryFile(suffix='.config',
prefix='robotium-',
dir=os.getcwd(),
delete=False)
fHandle.write("profile=%s\n" % self.remoteProfile)
fHandle.write("logfile=%s\n" % self.remoteLogFile)
fHandle.write("host=http://mochi.test:8888/tests\n")
fHandle.write("rawhost=http://%s:%s/tests\n" %
(self.options.remoteWebServer, self.options.httpPort))
if browserEnv:
envstr = ""
delim = ""
for key, value in browserEnv.items():
try:
value.index(',')
self.log.error(
"setupRobotiumConfig: browserEnv - Found a ',' "
"in our value, unable to process value. key=%s,value=%s"
% (key, value))
self.log.error("browserEnv=%s" % browserEnv)
except ValueError:
envstr += "%s%s=%s" % (delim, key, value)
delim = ","
fHandle.write("envvars=%s\n" % envstr)
fHandle.close()
self.device.rm(self.remoteConfigFile, force=True)
self.device.push(fHandle.name, self.remoteConfigFile)
os.unlink(fHandle.name)
def buildBrowserEnv(self):
"""
Return an environment dictionary suitable for remote use.
This is similar to buildBrowserEnv in runtestsremote.py.
"""
browserEnv = self.environment(xrePath=None, debugger=None)
# remove desktop environment not used on device
if "XPCOM_MEM_BLOAT_LOG" in browserEnv:
del browserEnv["XPCOM_MEM_BLOAT_LOG"]
browserEnv["MOZ_LOG_FILE"] = os.path.join(self.remoteMozLog,
self.mozLogName)
try:
browserEnv.update(
dict(
parse_key_value(self.options.environment,
context='--setenv')))
except KeyValueParseError as e:
self.log.error(str(e))
return None
return browserEnv
def runSingleTest(self, test):
"""
Run the specified test.
"""
self.log.debug("Running test %s" % test['name'])
self.mozLogName = "moz-%s.log" % test['name']
browserEnv = self.buildBrowserEnv()
self.setupRobotiumConfig(browserEnv)
self.setupRemoteProfile()
self.options.app = "am"
timeout = None
testName = test['name'].split('/')[-1].split('.java')[0]
if self.options.enable_coverage:
remoteCoverageFile = posixpath.join(
self.options.remoteTestRoot,
'robocop-coverage-%s.ec' % testName)
coverageFile = os.path.join(self.options.coverage_output_dir,
'robocop-coverage-%s.ec' % testName)
if self.options.autorun:
# This launches a test (using "am instrument") and instructs
# Fennec to /quit/ the browser (using Robocop:Quit) and to
# /finish/ all opened activities.
browserArgs = [
"instrument",
]
if self.options.enable_coverage:
browserArgs += [
"-e",
"coverage",
"true",
"-e",
"coverageFile",
remoteCoverageFile,
]
browserArgs += [
"-e",
"quit_and_finish",
"1",
"-e",
"deviceroot",
self.device.test_root,
"-e",
"class",
"org.mozilla.gecko.tests.%s" % testName,
"org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner",
]
else:
# This does not launch a test at all. It launches an activity
# that starts Fennec and then waits indefinitely, since cat
# never returns.
browserArgs = [
"start", "-n", "org.mozilla.roboexample.test/org.mozilla."
"gecko.LaunchFennecWithConfigurationActivity", "&&", "cat"
]
timeout = sys.maxint # Forever.
self.log.info("")
self.log.info(
"Serving mochi.test Robocop root at http://%s:%s/tests/robocop/"
% (self.options.remoteWebServer, self.options.httpPort))
self.log.info("")
result = -1
log_result = -1
try:
self.device.clear_logcat()
if not timeout:
timeout = self.options.timeout
if not timeout:
timeout = self.NO_OUTPUT_TIMEOUT
result, _ = self.auto.runApp(None,
browserEnv,
"am",
self.localProfile,
browserArgs,
timeout=timeout,
symbolsPath=self.options.symbolsPath)
self.log.debug("runApp completes with status %d" % result)
if result != 0:
self.log.error("runApp() exited with code %s" % result)
if self.device.is_file(self.remoteLogFile):
self.device.pull(self.remoteLogFile, self.localLog)
self.device.rm(self.remoteLogFile)
log_result = self.parseLocalLog()
if result != 0 or log_result != 0:
# Display remote diagnostics; if running in mach, keep output
# terse.
if self.options.log_mach is None:
self.printDeviceInfo(printLogcat=True)
if self.options.enable_coverage:
if self.device.is_file(remoteCoverageFile):
self.device.pull(remoteCoverageFile, coverageFile)
self.device.rm(remoteCoverageFile)
else:
self.log.warning(
"Code coverage output not found on remote device: %s" %
remoteCoverageFile)
except Exception:
self.log.error(
"Automation Error: Exception caught while running tests")
traceback.print_exc()
result = 1
self.log.debug("Test %s completes with status %d (log status %d)" %
(test['name'], int(result), int(log_result)))
return result
def runTests(self):
self.startup()
if isinstance(self.options.manifestFile, TestManifest):
mp = self.options.manifestFile
else:
mp = TestManifest(strict=False)
mp.read("robocop.ini")
filters = []
if self.options.totalChunks:
filters.append(
chunk_by_slice(self.options.thisChunk,
self.options.totalChunks))
robocop_tests = mp.active_tests(exists=False,
filters=filters,
**mozinfo.info)
if not self.options.autorun:
# Force a single loop iteration. The iteration will start Fennec and
# the httpd server, but not actually run a test.
self.options.test_paths = [robocop_tests[0]['name']]
active_tests = []
for test in robocop_tests:
if self.options.test_paths and test[
'name'] not in self.options.test_paths:
continue
if 'disabled' in test:
self.log.info('TEST-INFO | skipping %s | %s' %
(test['name'], test['disabled']))
continue
active_tests.append(test)
tests_by_manifest = defaultdict(list)
for test in active_tests:
tests_by_manifest[test['manifest']].append(test['name'])
self.log.suite_start(tests_by_manifest)
worstTestResult = None
for test in active_tests:
result = self.runSingleTest(test)
if worstTestResult is None or worstTestResult == 0:
worstTestResult = result
if worstTestResult is None:
self.log.warning(
"No tests run. Did you pass an invalid TEST_PATH?")
worstTestResult = 1
else:
print "INFO | runtests.py | Test summary: start."
logResult = self.logTestSummary()
print "INFO | runtests.py | Test summary: end."
if worstTestResult == 0:
worstTestResult = logResult
return worstTestResult
class RemoteCPPUnitTests(cppunittests.CPPUnitTests):
def __init__(self, options, progs):
cppunittests.CPPUnitTests.__init__(self)
self.options = options
self.device = ADBDevice(adb=options.adb_path or 'adb',
device=options.device_serial,
test_root=options.remote_test_root)
self.remote_test_root = posixpath.join(self.device.test_root,
"cppunittests")
self.remote_bin_dir = posixpath.join(self.remote_test_root, "b")
self.remote_tmp_dir = posixpath.join(self.remote_test_root, "tmp")
self.remote_home_dir = posixpath.join(self.remote_test_root, "h")
if options.setup:
self.setup_bin(progs)
def setup_bin(self, progs):
self.device.rm(self.remote_test_root, force=True, recursive=True)
self.device.mkdir(self.remote_home_dir, parents=True)
self.device.mkdir(self.remote_tmp_dir)
self.push_libs()
self.push_progs(progs)
self.device.chmod(self.remote_bin_dir, recursive=True, root=True)
def push_libs(self):
if self.options.local_apk:
with mozfile.TemporaryDirectory() as tmpdir:
apk_contents = ZipFile(self.options.local_apk)
for info in apk_contents.infolist():
if info.filename.endswith(".so"):
print >> sys.stderr, "Pushing %s.." % info.filename
remote_file = posixpath.join(
self.remote_bin_dir,
os.path.basename(info.filename))
apk_contents.extract(info, tmpdir)
local_file = os.path.join(tmpdir, info.filename)
with open(local_file) as f:
# Decompress xz-compressed file.
if f.read(5)[1:] == '7zXZ':
cmd = [
'xz', '-df', '--suffix', '.so', local_file
]
subprocess.check_output(cmd)
# xz strips the ".so" file suffix.
os.rename(local_file[:-3], local_file)
self.device.push(local_file, remote_file)
elif self.options.local_lib:
for file in os.listdir(self.options.local_lib):
if file.endswith(".so"):
print >> sys.stderr, "Pushing %s.." % file
remote_file = posixpath.join(self.remote_bin_dir, file)
local_file = os.path.join(self.options.local_lib, file)
self.device.push(local_file, remote_file)
# Additional libraries may be found in a sub-directory such as
# "lib/armeabi-v7a"
for subdir in ["assets", "lib"]:
local_arm_lib = os.path.join(self.options.local_lib, subdir)
if os.path.isdir(local_arm_lib):
for root, dirs, files in os.walk(local_arm_lib):
for file in files:
if (file.endswith(".so")):
print >> sys.stderr, "Pushing %s.." % file
remote_file = posixpath.join(
self.remote_bin_dir, file)
local_file = os.path.join(root, file)
self.device.push(local_file, remote_file)
def push_progs(self, progs):
for local_file in progs:
remote_file = posixpath.join(self.remote_bin_dir,
os.path.basename(local_file))
self.device.push(local_file, remote_file)
def build_environment(self):
env = self.build_core_environment()
env['LD_LIBRARY_PATH'] = self.remote_bin_dir
env["TMPDIR"] = self.remote_tmp_dir
env["HOME"] = self.remote_home_dir
env["MOZ_XRE_DIR"] = self.remote_bin_dir
if self.options.add_env:
for envdef in self.options.add_env:
envdef_parts = envdef.split("=", 1)
if len(envdef_parts) == 2:
env[envdef_parts[0]] = envdef_parts[1]
elif len(envdef_parts) == 1:
env[envdef_parts[0]] = ""
else:
self.log.warning("invalid --addEnv option skipped: %s" %
envdef)
return env
def run_one_test(self,
prog,
env,
symbols_path=None,
interactive=False,
timeout_factor=1):
"""
Run a single C++ unit test program remotely.
Arguments:
* prog: The path to the test program to run.
* env: The environment to use for running the program.
* symbols_path: A path to a directory containing Breakpad-formatted
symbol files for producing stack traces on crash.
* timeout_factor: An optional test-specific timeout multiplier.
Return True if the program exits with a zero status, False otherwise.
"""
basename = os.path.basename(prog)
remote_bin = posixpath.join(self.remote_bin_dir, basename)
self.log.test_start(basename)
test_timeout = cppunittests.CPPUnitTests.TEST_PROC_TIMEOUT * \
timeout_factor
try:
output = self.device.shell_output(remote_bin,
env=env,
cwd=self.remote_home_dir,
timeout=test_timeout)
returncode = 0
except ADBTimeoutError:
raise
except ADBProcessError as e:
output = e.adb_process.stdout
returncode = e.adb_process.exitcode
self.log.process_output(basename,
"\n%s" % output,
command=[remote_bin])
with mozfile.TemporaryDirectory() as tempdir:
self.device.pull(self.remote_home_dir, tempdir)
if mozcrash.check_for_crashes(tempdir,
symbols_path,
test_name=basename):
self.log.test_end(basename, status='CRASH', expected='PASS')
return False
result = returncode == 0
if not result:
self.log.test_end(basename,
status='FAIL',
expected='PASS',
message=("test failed with return code %s" %
returncode))
else:
self.log.test_end(basename, status='PASS', expected='PASS')
return result
class Raptor(object):
"""Container class for Raptor"""
def __init__(self,
app,
binary,
run_local=False,
obj_path=None,
gecko_profile=False,
gecko_profile_interval=None,
gecko_profile_entries=None,
symbols_path=None,
host=None,
power_test=False,
is_release_build=False,
debug_mode=False):
# Override the magic --host HOST_IP with the value of the environment variable.
if host == 'HOST_IP':
host = os.environ['HOST_IP']
self.config = {}
self.config['app'] = app
self.config['binary'] = binary
self.config['platform'] = mozinfo.os
self.config['processor'] = mozinfo.processor
self.config['run_local'] = run_local
self.config['obj_path'] = obj_path
self.config['gecko_profile'] = gecko_profile
self.config['gecko_profile_interval'] = gecko_profile_interval
self.config['gecko_profile_entries'] = gecko_profile_entries
self.config['symbols_path'] = symbols_path
self.config['host'] = host
self.config['power_test'] = power_test
self.config['is_release_build'] = is_release_build
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
self.log = get_default_logger(component='raptor-main')
self.control_server = None
self.playback = None
self.benchmark = None
self.gecko_profiler = None
self.post_startup_delay = 30000
self.device = None
# debug mode is currently only supported when running locally
self.debug_mode = debug_mode if self.config['run_local'] else False
# if running debug-mode reduce the pause after browser startup
if self.debug_mode:
self.post_startup_delay = 3000
self.log.info(
"debug-mode enabled, reducing post-browser startup pause to %d ms"
% self.post_startup_delay)
# Create the profile; for geckoview/fennec we want a firefox profile type
if self.config['app'] in ["geckoview", "fennec"]:
self.profile = create_profile('firefox')
else:
self.profile = create_profile(self.config['app'])
# Merge in base profiles
with open(os.path.join(self.profile_data_dir, 'profiles.json'),
'r') as fh:
base_profiles = json.load(fh)['raptor']
for name in base_profiles:
path = os.path.join(self.profile_data_dir, name)
self.log.info("Merging profile: {}".format(path))
self.profile.merge(path)
# add profile dir to our config
self.config['local_profile_dir'] = self.profile.profile
# create results holder
self.results_handler = RaptorResultsHandler()
# when testing desktop browsers we use mozrunner to start the browser; when
# testing on android (i.e. geckoview) we use mozdevice to control the device app
if self.config['app'] in ["geckoview", "fennec"]:
# create the android device handler; it gets initiated and sets up adb etc
self.log.info("creating android device handler using mozdevice")
self.device = ADBDevice(verbose=True)
self.device.clear_logcat()
if self.config['power_test']:
init_geckoview_power_test(self)
else:
# create the desktop browser runner
self.log.info("creating browser runner using mozrunner")
self.output_handler = OutputHandler()
process_args = {
'processOutputLine': [self.output_handler],
}
runner_cls = runners[app]
self.runner = runner_cls(binary,
profile=self.profile,
process_args=process_args,
symbols_path=self.config['symbols_path'])
self.log.info("raptor config: %s" % str(self.config))
@property
def profile_data_dir(self):
if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
'testing', 'profiles')
if build:
return os.path.join(build.topsrcdir, 'testing', 'profiles')
return os.path.join(here, 'profile_data')
def start_control_server(self):
self.control_server = RaptorControlServer(self.results_handler,
self.debug_mode)
self.control_server.start()
# for android we must make the control server available to the device
if self.config['app'] in ['geckoview', 'fennec'] and \
self.config['host'] in ('localhost', '127.0.0.1'):
self.log.info(
"making the raptor control server port available to device")
_tcp_port = "tcp:%s" % self.control_server.port
self.device.create_socket_connection('reverse', _tcp_port,
_tcp_port)
def get_playback_config(self, test):
self.config['playback_tool'] = test.get('playback')
self.log.info("test uses playback tool: %s " %
self.config['playback_tool'])
self.config['playback_binary_manifest'] = test.get(
'playback_binary_manifest', None)
_key = 'playback_binary_zip_%s' % self.config['platform']
self.config['playback_binary_zip'] = test.get(_key, None)
self.config['playback_pageset_manifest'] = test.get(
'playback_pageset_manifest', None)
_key = 'playback_pageset_zip_%s' % self.config['platform']
self.config['playback_pageset_zip'] = test.get(_key, None)
self.config['playback_recordings'] = test.get('playback_recordings',
None)
self.config['python3_win_manifest'] = test.get('python3_win_manifest',
None)
def run_test(self, test, timeout=None):
self.log.info("starting raptor test: %s" % test['name'])
self.log.info("test settings: %s" % str(test))
self.log.info("raptor config: %s" % str(self.config))
# benchmark-type tests require the benchmark test to be served out
if test.get('type') == "benchmark":
self.benchmark = Benchmark(self.config, test)
benchmark_port = int(self.benchmark.port)
# for android we must make the benchmarks server available to the device
if self.config['app'] in ['geckoview', 'fennec'] and \
self.config['host'] in ('localhost', '127.0.0.1'):
self.log.info(
"making the raptor benchmarks server port available to device"
)
_tcp_port = "tcp:%s" % benchmark_port
self.device.create_socket_connection('reverse', _tcp_port,
_tcp_port)
else:
benchmark_port = 0
gen_test_config(self.config['app'],
test['name'],
self.control_server.port,
self.post_startup_delay,
host=self.config['host'],
b_port=benchmark_port,
debug_mode=1 if self.debug_mode else 0)
# must intall raptor addon each time because we dynamically update some content
# note: for chrome the addon is just a list of paths that ultimately are added
# to the chromium command line '--load-extension' argument
raptor_webext = os.path.join(webext_dir, 'raptor')
self.log.info("installing webext %s" % raptor_webext)
self.profile.addons.install(raptor_webext)
# add test specific preferences
if test.get("preferences", None) is not None:
if self.config['app'] == "firefox":
self.profile.set_preferences(json.loads(test['preferences']))
else:
self.log.info("preferences were configured for the test, \
but we do not install them on non Firefox browsers."
)
# if 'alert_on' was provided in the test INI, we must add that to our config
# for use in our results.py and output.py
# test['alert_on'] has already been converted to a list and stripped of spaces
self.config['subtest_alert_on'] = test.get('alert_on', None)
# on firefox we can get an addon id; chrome addon actually is just cmd line arg
if self.config['app'] in ['firefox', 'geckoview', 'fennec']:
webext_id = self.profile.addons.addon_details(raptor_webext)['id']
# for android/geckoview, create a top-level raptor folder on the device
# sdcard; if it already exists remove it so we start fresh each time
if self.config['app'] in ["geckoview", "fennec"]:
self.device_raptor_dir = "/sdcard/raptor"
self.config['device_raptor_dir'] = self.device_raptor_dir
if self.device.is_dir(self.device_raptor_dir):
self.log.info("deleting existing device raptor dir: %s" %
self.device_raptor_dir)
self.device.rm(self.device_raptor_dir, recursive=True)
self.log.info("creating raptor folder on sdcard: %s" %
self.device_raptor_dir)
self.device.mkdir(self.device_raptor_dir)
self.device.chmod(self.device_raptor_dir, recursive=True)
# some tests require tools to playback the test pages
if test.get('playback', None) is not None:
# startup the playback tool
self.get_playback_config(test)
self.playback = get_playback(self.config, self.device)
# for android we must make the playback server available to the device
if self.config['app'] == "geckoview" and self.config['host'] \
in ('localhost', '127.0.0.1'):
self.log.info(
"making the raptor playback server port available to device"
)
_tcp_port = "tcp:8080"
self.device.create_socket_connection('reverse', _tcp_port,
_tcp_port)
if self.config['app'] in ('geckoview', 'firefox', 'fennec') and \
self.config['host'] not in ('localhost', '127.0.0.1'):
# Must delete the proxy settings from the profile if running
# the test with a host different from localhost.
userjspath = os.path.join(self.profile.profile, 'user.js')
with open(userjspath) as userjsfile:
prefs = userjsfile.readlines()
prefs = [pref for pref in prefs if 'network.proxy' not in pref]
with open(userjspath, 'w') as userjsfile:
userjsfile.writelines(prefs)
# for geckoview/android pageload playback we can't use a policy to turn on the
# proxy; we need to set prefs instead; note that the 'host' may be different
# than '127.0.0.1' so we must set the prefs accordingly
if self.config['app'] == "geckoview" and test.get('playback',
None) is not None:
self.log.info(
"setting profile prefs to turn on the geckoview browser proxy")
no_proxies_on = "localhost, 127.0.0.1, %s" % self.config['host']
proxy_prefs = {}
proxy_prefs["network.proxy.type"] = 1
proxy_prefs["network.proxy.http"] = self.config['host']
proxy_prefs["network.proxy.http_port"] = 8080
proxy_prefs["network.proxy.ssl"] = self.config['host']
proxy_prefs["network.proxy.ssl_port"] = 8080
proxy_prefs["network.proxy.no_proxies_on"] = no_proxies_on
self.profile.set_preferences(proxy_prefs)
# now some final settings, and then startup of the browser under test
if self.config['app'] in ["geckoview", "fennec"]:
# for geckoview/fennec we must copy the profile onto the device and set perms
if not self.device.is_app_installed(self.config['binary']):
raise Exception('%s is not installed' % self.config['binary'])
self.device_profile = os.path.join(self.device_raptor_dir,
"profile")
if self.device.is_dir(self.device_profile):
self.log.info("deleting existing device profile folder: %s" %
self.device_profile)
self.device.rm(self.device_profile, recursive=True)
self.log.info("creating profile folder on device: %s" %
self.device_profile)
self.device.mkdir(self.device_profile)
self.log.info("copying firefox profile onto the device")
self.log.info("note: the profile folder being copied is: %s" %
self.profile.profile)
self.log.info(
'the adb push cmd copies that profile dir to a new temp dir before copy'
)
self.device.push(self.profile.profile, self.device_profile)
self.device.chmod(self.device_profile, recursive=True)
# now start the geckoview/fennec app
self.log.info("starting %s" % self.config['app'])
extra_args = [
"-profile", self.device_profile, "--es", "env0",
"LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6"
]
if self.config['app'] == 'geckoview':
# launch geckoview example app
try:
# make sure the geckoview app is not running before
# attempting to start.
self.device.stop_application(self.config['binary'])
self.device.launch_activity(self.config['binary'],
"GeckoViewActivity",
extra_args=extra_args,
url='about:blank',
e10s=True,
fail_if_running=False)
except Exception:
self.log.error("Exception launching %s" %
self.config['binary'])
if self.config['power_test']:
finish_geckoview_power_test(self)
raise
else:
# launch fennec
try:
# if fennec is already running, shut it down first
self.device.stop_application(self.config['binary'])
self.device.launch_fennec(self.config['binary'],
extra_args=extra_args,
url='about:blank',
fail_if_running=False)
except Exception:
self.log.error("Exception launching %s" %
self.config['binary'])
if self.config['power_test']:
finish_geckoview_power_test(self)
raise
self.control_server.device = self.device
self.control_server.app_name = self.config['binary']
else:
# For Firefox we need to set
# MOZ_DISABLE_NONLOCAL_CONNECTIONS=1 env var before
# startup when testing release builds from mozilla-beta or
# mozilla-release. This is because of restrictions on
# release builds that require webextensions to be signed
# unless MOZ_DISABLE_NONLOCAL_CONNECTIONS is set to '1'.
if self.config['app'] == "firefox" and self.config[
'is_release_build']:
self.log.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=1")
os.environ['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = "1"
# if running debug-mode, tell Firefox to open the browser console on startup
# for google chrome, open the devtools on the raptor test tab
if self.debug_mode:
if self.config['app'] == "firefox":
self.runner.cmdargs.extend(['-jsconsole'])
if self.config['app'] == "chrome":
self.runner.cmdargs.extend(
['--auto-open-devtools-for-tabs'])
# now start the desktop browser
self.log.info("starting %s" % self.config['app'])
# if running a pageload test on google chrome, add the cmd line options
# to turn on the proxy and ignore security certificate errors
# if using host localhost, 127.0.0.1.
if self.config['app'] == "chrome" and test.get('playback',
None) is not None:
chrome_args = [
'--proxy-server=127.0.0.1:8080',
'--proxy-bypass-list=localhost;127.0.0.1',
'--ignore-certificate-errors',
'--no-default-browser-check',
]
if self.config['host'] not in ('localhost', '127.0.0.1'):
chrome_args[0] = chrome_args[0].replace(
'127.0.0.1', self.config['host'])
if ' '.join(chrome_args) not in ' '.join(self.runner.cmdargs):
self.runner.cmdargs.extend(chrome_args)
self.runner.start()
proc = self.runner.process_handler
self.output_handler.proc = proc
self.control_server.browser_proc = proc
# pageload tests need to be able to access non-local connections via mitmproxy
if self.config['app'] == "firefox" and self.config['is_release_build'] and \
test.get('playback', None) is not None:
self.log.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=0")
os.environ['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = "0"
# if geckoProfile is enabled, initialize it
if self.config['gecko_profile'] is True:
self._init_gecko_profiling(test)
# tell the control server the gecko_profile dir; the control server will
# receive the actual gecko profiles from the web ext and will write them
# to disk; then profiles are picked up by gecko_profile.symbolicate
self.control_server.gecko_profile_dir = self.gecko_profiler.gecko_profile_dir
# set our cs flag to indicate we are running the browser/app
self.control_server._finished = False
# convert to seconds and account for page cycles
timeout = int(timeout / 1000) * int(test['page_cycles'])
# account for the pause the raptor webext runner takes after browser startup
timeout += int(self.post_startup_delay / 1000)
# if geckoProfile enabled, give browser more time for profiling
if self.config['gecko_profile'] is True:
timeout += 5 * 60
try:
elapsed_time = 0
while not self.control_server._finished:
time.sleep(1)
# we only want to force browser-shutdown on timeout if not in debug mode;
# in debug-mode we leave the browser running (require manual shutdown)
if not self.debug_mode:
elapsed_time += 1
if elapsed_time > (timeout) - 5: # stop 5 seconds early
self.log.info(
"application timed out after {} seconds".format(
timeout))
self.control_server.wait_for_quit()
break
finally:
if self.config['app'] == "geckoview":
if self.config['power_test']:
finish_geckoview_power_test(self)
self.check_for_crashes()
if self.playback is not None:
self.playback.stop()
# remove the raptor webext; as it must be reloaded with each subtest anyway
self.log.info("removing webext %s" % raptor_webext)
if self.config['app'] in ['firefox', 'geckoview', 'fennec']:
self.profile.addons.remove_addon(webext_id)
# for chrome the addon is just a list (appended to cmd line)
if self.config['app'] in ["chrome", "chrome-android"]:
self.profile.addons.remove(raptor_webext)
# gecko profiling symbolication
if self.config['gecko_profile'] is True:
self.gecko_profiler.symbolicate()
# clean up the temp gecko profiling folders
self.log.info("cleaning up after gecko profiling")
self.gecko_profiler.clean()
# browser should be closed by now but this is a backup-shutdown (if not in debug-mode)
if not self.debug_mode:
if self.config['app'] not in ['geckoview', 'fennec']:
if self.runner.is_running():
self.runner.stop()
# TODO the geckoview app should have been shutdown by this point by the
# control server, but we can double-check here to make sure
else:
# in debug mode, and running locally, leave the browser running
if self.config['run_local']:
self.log.info(
"* debug-mode enabled - please shutdown the browser manually..."
)
self.runner.wait(timeout=None)
def _init_gecko_profiling(self, test):
self.log.info("initializing gecko profiler")
upload_dir = os.getenv('MOZ_UPLOAD_DIR')
if not upload_dir:
self.log.critical(
"Profiling ignored because MOZ_UPLOAD_DIR was not set")
else:
self.gecko_profiler = GeckoProfile(upload_dir, self.config, test)
def process_results(self, test_names):
# when running locally output results in build/raptor.json; when running
# in production output to a local.json to be turned into tc job artifact
if self.config.get('run_local', False):
if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
raptor_json_path = os.path.join(
os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing',
'mozharness', 'build', 'raptor.json')
else:
raptor_json_path = os.path.join(here, 'raptor.json')
else:
raptor_json_path = os.path.join(os.getcwd(), 'local.json')
self.config['raptor_json_path'] = raptor_json_path
return self.results_handler.summarize_and_output(
self.config, test_names)
def get_page_timeout_list(self):
return self.results_handler.page_timeout_list
def check_for_crashes(self):
if self.config['app'] in ["geckoview", "fennec"]:
# Turn off verbose to prevent logcat from being inserted into the main log.
verbose = self.device._verbose
self.device._verbose = False
logcat = self.device.get_logcat()
self.device._verbose = verbose
if logcat:
if mozcrash.check_for_java_exception(logcat, "raptor"):
return
try:
dump_dir = tempfile.mkdtemp()
remote_dir = posixpath.join(self.device_profile, 'minidumps')
if not self.device.is_dir(remote_dir):
self.log.error(
"No crash directory (%s) found on remote device" %
remote_dir)
return
self.device.pull(remote_dir, dump_dir)
mozcrash.log_crashes(self.log, dump_dir,
self.config['symbols_path'])
finally:
try:
shutil.rmtree(dump_dir)
except Exception:
self.log.warning("unable to remove directory: %s" %
dump_dir)
else:
try:
self.runner.check_for_crashes()
except NotImplementedError: # not implemented for Chrome
pass
def clean_up(self):
self.control_server.stop()
if self.config['app'] not in ['geckoview', 'fennec']:
self.runner.stop()
elif self.config['app'] in ['geckoview', 'fennec']:
self.log.info('removing reverse socket connections')
self.device.remove_socket_connections('reverse')
else:
pass
self.log.info("finished")
