class RemoteReftest(RefTest): use_marionette = False resolver_cls = RemoteReftestResolver def __init__(self, options, scriptDir): RefTest.__init__(self, options.suite) self.run_by_manifest = False self.scriptDir = scriptDir self.localLogName = options.localLogName verbose = False if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug': verbose = True print "set verbose!" self.device = ADBAndroid(adb=options.adb_path, device=options.deviceSerial, test_root=options.remoteTestRoot, verbose=verbose) if options.remoteTestRoot is None: options.remoteTestRoot = posixpath.join(self.device.test_root, "reftest") options.remoteProfile = posixpath.join(options.remoteTestRoot, "profile") options.remoteLogFile = posixpath.join(options.remoteTestRoot, "reftest.log") options.logFile = options.remoteLogFile self.remoteProfile = options.remoteProfile self.remoteTestRoot = options.remoteTestRoot if not options.ignoreWindowSize: parts = self.device.get_info('screen')['screen'][0].split() width = int(parts[0].split(':')[1]) height = int(parts[1].split(':')[1]) if (width < 1366 or height < 1050): self.error("ERROR: Invalid screen resolution %sx%s, " "please adjust to 1366x1050 or higher" % (width, height)) self._populate_logger(options) self.outputHandler = OutputHandler(self.log, options.utilityPath, options.symbolsPath) # RemoteAutomation.py's 'messageLogger' is also used by mochitest. Mimic a mochitest # MessageLogger object to re-use this code path. self.outputHandler.write = self.outputHandler.__call__ self.automation = RemoteAutomation(self.device, options.app, self.remoteProfile, options.remoteLogFile, processArgs=None) self.automation._processArgs['messageLogger'] = self.outputHandler self.environment = self.automation.environment if self.automation.IS_DEBUG_BUILD: self.SERVER_STARTUP_TIMEOUT = 180 else: self.SERVER_STARTUP_TIMEOUT = 90 self.remoteCache = os.path.join(options.remoteTestRoot, "cache/") # Check that Firefox is installed expected = options.app.split('/')[-1] if not self.device.is_app_installed(expected): raise Exception("%s is not installed on this device" % expected) self.automation.deleteANRs() self.automation.deleteTombstones() self.device.clear_logcat() self.device.rm(self.remoteCache, force=True, recursive=True) procName = options.app.split('/')[-1] self.device.pkill(procName) if self.device.process_exist(procName): self.log.error("unable to kill %s before starting tests!" % procName) def findPath(self, paths, filename=None): for path in paths: p = path if filename: p = os.path.join(p, filename) if os.path.exists(self.getFullPath(p)): return path return None def startWebServer(self, options): """ Create the webserver on the host and start it up """ remoteXrePath = options.xrePath remoteUtilityPath = options.utilityPath localAutomation = Automation() localAutomation.IS_WIN32 = False localAutomation.IS_LINUX = False localAutomation.IS_MAC = False localAutomation.UNIXISH = False hostos = sys.platform if (hostos == 'mac' or hostos == 'darwin'): localAutomation.IS_MAC = True elif (hostos == 'linux' or hostos == 'linux2'): localAutomation.IS_LINUX = True localAutomation.UNIXISH = True elif (hostos == 'win32' or hostos == 'win64'): localAutomation.BIN_SUFFIX = ".exe" localAutomation.IS_WIN32 = True paths = [options.xrePath, localAutomation.DIST_BIN] options.xrePath = self.findPath(paths) if options.xrePath is None: print( "ERROR: unable to find xulrunner path for %s, " "please specify with --xre-path" % (os.name)) return 1 paths.append("bin") paths.append(os.path.join("..", "bin")) xpcshell = "xpcshell" if (os.name == "nt"): xpcshell += ".exe" if (options.utilityPath): paths.insert(0, options.utilityPath) options.utilityPath = self.findPath(paths, xpcshell) if options.utilityPath is None: print( "ERROR: unable to find utility path for %s, " "please specify with --utility-path" % (os.name)) return 1 options.serverProfilePath = tempfile.mkdtemp() self.server = ReftestServer(localAutomation, options, self.scriptDir) retVal = self.server.start() if retVal: return retVal retVal = self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT) if retVal: return retVal options.xrePath = remoteXrePath options.utilityPath = remoteUtilityPath return 0 def stopWebServer(self, options): self.server.stop() def killNamedProc(self, pname, orphans=True): """ Kill processes matching the given command name """ self.log.info("Checking for %s processes..." % pname) for proc in psutil.process_iter(): try: if proc.name() == pname: procd = proc.as_dict( attrs=['pid', 'ppid', 'name', 'username']) if proc.ppid() == 1 or not orphans: self.log.info("killing %s" % procd) try: os.kill(proc.pid, getattr(signal, "SIGKILL", signal.SIGTERM)) except Exception as e: self.log.info("Failed to kill process %d: %s" % (proc.pid, str(e))) else: self.log.info("NOT killing %s (not an orphan?)" % procd) except Exception: # may not be able to access process info for all processes continue def createReftestProfile(self, options, **kwargs): profile = RefTest.createReftestProfile(self, options, server=options.remoteWebServer, port=options.httpPort, **kwargs) profileDir = profile.profile prefs = {} prefs["app.update.url.android"] = "" prefs["browser.firstrun.show.localepicker"] = False prefs["reftest.remote"] = True prefs[ "datareporting.policy.dataSubmissionPolicyBypassAcceptance"] = True # move necko cache to a location that can be cleaned up prefs["browser.cache.disk.parent_directory"] = self.remoteCache prefs["layout.css.devPixelsPerPx"] = "1.0" # Because Fennec is a little wacky (see bug 1156817) we need to load the # reftest pages at 1.0 zoom, rather than zooming to fit the CSS viewport. prefs["apz.allow_zooming"] = False # Set the extra prefs. profile.set_preferences(prefs) try: self.device.push(profileDir, options.remoteProfile) self.device.chmod(options.remoteProfile, recursive=True) except Exception: print "Automation Error: Failed to copy profiledir to device" raise return profile def copyExtraFilesToProfile(self, options, profile): profileDir = profile.profile RefTest.copyExtraFilesToProfile(self, options, profile) if len(os.listdir(profileDir)) > 0: try: self.device.push(profileDir, options.remoteProfile) self.device.chmod(options.remoteProfile, recursive=True) except Exception: print "Automation Error: Failed to copy extra files to device" raise def printDeviceInfo(self, printLogcat=False): try: if printLogcat: logcat = self.device.get_logcat( filter_out_regexps=fennecLogcatFilters) print ''.join(logcat) print "Device info:" devinfo = self.device.get_info() for category in devinfo: if type(devinfo[category]) is list: print " %s:" % category for item in devinfo[category]: print " %s" % item else: print " %s: %s" % (category, devinfo[category]) print "Test root: %s" % self.device.test_root except Exception as e: print "WARNING: Error getting device information: %s" % str(e) def environment(self, **kwargs): return self.automation.environment(**kwargs) def buildBrowserEnv(self, options, profileDir): browserEnv = RefTest.buildBrowserEnv(self, options, profileDir) # remove desktop environment not used on device if "XPCOM_MEM_BLOAT_LOG" in browserEnv: del browserEnv["XPCOM_MEM_BLOAT_LOG"] return browserEnv def runApp(self, options, cmdargs=None, timeout=None, debuggerInfo=None, symbolsPath=None, valgrindPath=None, valgrindArgs=None, valgrindSuppFiles=None, **profileArgs): if cmdargs is None: cmdargs = [] if self.use_marionette: cmdargs.append('-marionette') binary = options.app profile = self.createReftestProfile(options, **profileArgs) # browser environment env = self.buildBrowserEnv(options, profile.profile) self.log.info("Running with e10s: {}".format(options.e10s)) status, self.lastTestSeen = self.automation.runApp( None, env, binary, profile.profile, cmdargs, utilityPath=options.utilityPath, xrePath=options.xrePath, debuggerInfo=debuggerInfo, symbolsPath=symbolsPath, timeout=timeout) self.cleanup(profile.profile) return status def cleanup(self, profileDir): self.device.rm(self.remoteTestRoot, force=True, recursive=True) self.device.rm(self.remoteProfile, force=True, recursive=True) self.device.rm(self.remoteCache, force=True, recursive=True) RefTest.cleanup(self, profileDir)
class MochiRemote(MochitestDesktop): localProfile = None logMessages = [] def __init__(self, options): MochitestDesktop.__init__(self, options.flavor, vars(options)) verbose = False if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug': verbose = True if hasattr(options, 'log'): delattr(options, 'log') self.certdbNew = True self.chromePushed = False self.mozLogName = "moz.log" self.device = ADBDevice(adb=options.adbPath or 'adb', device=options.deviceSerial, test_root=options.remoteTestRoot, verbose=verbose) if options.remoteTestRoot is None: options.remoteTestRoot = self.device.test_root options.dumpOutputDirectory = options.remoteTestRoot self.remoteLogFile = posixpath.join(options.remoteTestRoot, "logs", "mochitest.log") logParent = posixpath.dirname(self.remoteLogFile) self.device.rm(logParent, force=True, recursive=True) self.device.mkdir(logParent) self.remoteProfile = posixpath.join(options.remoteTestRoot, "profile/") self.device.rm(self.remoteProfile, force=True, recursive=True) self.counts = dict() self.message_logger = MessageLogger(logger=None) self.message_logger.logger = self.log process_args = {'messageLogger': self.message_logger, 'counts': self.counts} self.automation = RemoteAutomation(self.device, options.remoteappname, self.remoteProfile, self.remoteLogFile, processArgs=process_args) self.environment = self.automation.environment # Check that Firefox is installed expected = options.app.split('/')[-1] if not self.device.is_app_installed(expected): raise Exception("%s is not installed on this device" % expected) self.automation.deleteANRs() self.automation.deleteTombstones() self.device.clear_logcat() self.remoteModulesDir = posixpath.join(options.remoteTestRoot, "modules/") self.remoteCache = posixpath.join(options.remoteTestRoot, "cache/") self.device.rm(self.remoteCache, force=True, recursive=True) # move necko cache to a location that can be cleaned up options.extraPrefs += ["browser.cache.disk.parent_directory=%s" % self.remoteCache] self.remoteMozLog = posixpath.join(options.remoteTestRoot, "mozlog") self.device.rm(self.remoteMozLog, force=True, recursive=True) self.device.mkdir(self.remoteMozLog) self.remoteChromeTestDir = posixpath.join( options.remoteTestRoot, "chrome") self.device.rm(self.remoteChromeTestDir, force=True, recursive=True) self.device.mkdir(self.remoteChromeTestDir) procName = options.app.split('/')[-1] self.device.stop_application(procName) if self.device.process_exist(procName): self.log.warning("unable to kill %s before running tests!" % procName) # Add Android version (SDK level) to mozinfo so that manifest entries # can be conditional on android_version. self.log.info( "Android sdk version '%s'; will use this to filter manifests" % str(self.device.version)) mozinfo.info['android_version'] = str(self.device.version) mozinfo.info['isFennec'] = not ('geckoview' in options.app) mozinfo.info['is_emulator'] = self.device._device_serial.startswith('emulator-') def cleanup(self, options, final=False): if final: self.device.rm(self.remoteChromeTestDir, force=True, recursive=True) self.chromePushed = False uploadDir = os.environ.get('MOZ_UPLOAD_DIR', None) if uploadDir and self.device.is_dir(self.remoteMozLog): self.device.pull(self.remoteMozLog, uploadDir) self.device.rm(self.remoteLogFile, force=True) self.device.rm(self.remoteProfile, force=True, recursive=True) self.device.rm(self.remoteCache, force=True, recursive=True) MochitestDesktop.cleanup(self, options, final) self.localProfile = None def dumpScreen(self, utilityPath): if self.haveDumpedScreen: self.log.info( "Not taking screenshot here: see the one that was previously logged") return self.haveDumpedScreen = True if self.device._device_serial.startswith('emulator-'): dump_screen(utilityPath, self.log) else: dump_device_screen(self.device, self.log) def findPath(self, paths, filename=None): for path in paths: p = path if filename: p = os.path.join(p, filename) if os.path.exists(self.getFullPath(p)): return path return None def makeLocalAutomation(self): localAutomation = Automation() localAutomation.IS_WIN32 = False localAutomation.IS_LINUX = False localAutomation.IS_MAC = False localAutomation.UNIXISH = False hostos = sys.platform if (hostos == 'mac' or hostos == 'darwin'): localAutomation.IS_MAC = True elif (hostos == 'linux' or hostos == 'linux2'): localAutomation.IS_LINUX = True localAutomation.UNIXISH = True elif (hostos == 'win32' or hostos == 'win64'): localAutomation.BIN_SUFFIX = ".exe" localAutomation.IS_WIN32 = True return localAutomation # This seems kludgy, but this class uses paths from the remote host in the # options, except when calling up to the base class, which doesn't # understand the distinction. This switches out the remote values for local # ones that the base class understands. This is necessary for the web # server, SSL tunnel and profile building functions. def switchToLocalPaths(self, options): """ Set local paths in the options, return a function that will restore remote values """ remoteXrePath = options.xrePath remoteProfilePath = options.profilePath remoteUtilityPath = options.utilityPath localAutomation = self.makeLocalAutomation() paths = [ options.xrePath, localAutomation.DIST_BIN, ] options.xrePath = self.findPath(paths) if options.xrePath is None: self.log.error( "unable to find xulrunner path for %s, please specify with --xre-path" % os.name) sys.exit(1) xpcshell = "xpcshell" if (os.name == "nt"): xpcshell += ".exe" if options.utilityPath: paths = [options.utilityPath, options.xrePath] else: paths = [options.xrePath] options.utilityPath = self.findPath(paths, xpcshell) if options.utilityPath is None: self.log.error( "unable to find utility path for %s, please specify with --utility-path" % os.name) sys.exit(1) xpcshell_path = os.path.join(options.utilityPath, xpcshell) if localAutomation.elf_arm(xpcshell_path): self.log.error('xpcshell at %s is an ARM binary; please use ' 'the --utility-path argument to specify the path ' 'to a desktop version.' % xpcshell_path) sys.exit(1) if self.localProfile: options.profilePath = self.localProfile else: options.profilePath = None def fixup(): options.xrePath = remoteXrePath options.utilityPath = remoteUtilityPath options.profilePath = remoteProfilePath return fixup def startServers(self, options, debuggerInfo): """ Create the servers on the host and start them up """ restoreRemotePaths = self.switchToLocalPaths(options) MochitestDesktop.startServers( self, options, debuggerInfo) restoreRemotePaths() def buildProfile(self, options): restoreRemotePaths = self.switchToLocalPaths(options) if options.testingModulesDir: try: self.device.push(options.testingModulesDir, self.remoteModulesDir) self.device.chmod(self.remoteModulesDir, recursive=True, root=True) except Exception: self.log.error( "Automation Error: Unable to copy test modules to device.") raise savedTestingModulesDir = options.testingModulesDir options.testingModulesDir = self.remoteModulesDir else: savedTestingModulesDir = None manifest = MochitestDesktop.buildProfile(self, options) if savedTestingModulesDir: options.testingModulesDir = savedTestingModulesDir self.localProfile = options.profilePath restoreRemotePaths() options.profilePath = self.remoteProfile return manifest def buildURLOptions(self, options, env): saveLogFile = options.logFile options.logFile = self.remoteLogFile options.profilePath = self.localProfile env["MOZ_HIDE_RESULTS_TABLE"] = "1" retVal = MochitestDesktop.buildURLOptions(self, options, env) # we really need testConfig.js (for browser chrome) try: self.device.push(options.profilePath, self.remoteProfile) self.device.chmod(self.remoteProfile, recursive=True, root=True) except Exception: self.log.error("Automation Error: Unable to copy profile to device.") raise options.profilePath = self.remoteProfile options.logFile = saveLogFile return retVal def getChromeTestDir(self, options): local = super(MochiRemote, self).getChromeTestDir(options) remote = self.remoteChromeTestDir if options.flavor == 'chrome' and not self.chromePushed: self.log.info("pushing %s to %s on device..." % (local, remote)) local = os.path.join(local, "chrome") self.device.push(local, remote) self.chromePushed = True return remote def getLogFilePath(self, logFile): return logFile def printDeviceInfo(self, printLogcat=False): try: if printLogcat: logcat = self.device.get_logcat( filter_out_regexps=fennecLogcatFilters) for l in logcat: ul = l.decode('utf-8', errors='replace') sl = ul.encode('iso8859-1', errors='replace') self.log.info(sl) self.log.info("Device info:") devinfo = self.device.get_info() for category in devinfo: if type(devinfo[category]) is list: self.log.info(" %s:" % category) for item in devinfo[category]: self.log.info(" %s" % item) else: self.log.info(" %s: %s" % (category, devinfo[category])) self.log.info("Test root: %s" % self.device.test_root) except ADBTimeoutError: raise except Exception as e: self.log.warning("Error getting device information: %s" % str(e)) def getGMPPluginPath(self, options): # TODO: bug 1149374 return None def buildBrowserEnv(self, options, debugger=False): browserEnv = MochitestDesktop.buildBrowserEnv( self, options, debugger=debugger) # remove desktop environment not used on device if "XPCOM_MEM_BLOAT_LOG" in browserEnv: del browserEnv["XPCOM_MEM_BLOAT_LOG"] # override mozLogs to avoid processing in MochitestDesktop base class self.mozLogs = None browserEnv["MOZ_LOG_FILE"] = os.path.join( self.remoteMozLog, self.mozLogName) if options.dmd: browserEnv['DMD'] = '1' # Contents of remoteMozLog will be pulled from device and copied to the # host MOZ_UPLOAD_DIR, to be made available as test artifacts. Make # MOZ_UPLOAD_DIR available to the browser environment so that tests # can use it as though they were running on the host. browserEnv["MOZ_UPLOAD_DIR"] = self.remoteMozLog return browserEnv def runApp(self, *args, **kwargs): """front-end automation.py's `runApp` functionality until FennecRunner is written""" # automation.py/remoteautomation `runApp` takes the profile path, # whereas runtest.py's `runApp` takes a mozprofile object. if 'profileDir' not in kwargs and 'profile' in kwargs: kwargs['profileDir'] = kwargs.pop('profile').profile # remove args not supported by automation.py kwargs.pop('marionette_args', None) ret, _ = self.automation.runApp(*args, **kwargs) self.countpass += self.counts['pass'] self.countfail += self.counts['fail'] self.counttodo += self.counts['todo'] return ret, None
class RobocopTestRunner(MochitestDesktop): """ A test harness for Robocop. Robocop tests are UI tests for Firefox for Android, based on the Robotium test framework. This harness leverages some functionality from mochitest, for convenience. """ # Some robocop tests run for >60 seconds without generating any output. NO_OUTPUT_TIMEOUT = 180 def __init__(self, options, message_logger): """ Simple one-time initialization. """ MochitestDesktop.__init__(self, options.flavor, vars(options)) verbose = False if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug': verbose = True self.device = ADBDevice(adb=options.adbPath or 'adb', device=options.deviceSerial, test_root=options.remoteTestRoot, verbose=verbose) # Check that Firefox is installed expected = options.app.split('/')[-1] if not self.device.is_app_installed(expected): raise Exception("%s is not installed on this device" % expected) options.logFile = "robocop.log" if options.remoteTestRoot is None: options.remoteTestRoot = self.device.test_root self.remoteProfile = posixpath.join(options.remoteTestRoot, "profile") self.remoteProfileCopy = posixpath.join(options.remoteTestRoot, "profile-copy") self.remoteModulesDir = posixpath.join(options.remoteTestRoot, "modules/") self.remoteConfigFile = posixpath.join(options.remoteTestRoot, "robotium.config") self.remoteLogFile = posixpath.join(options.remoteTestRoot, "logs", "robocop.log") self.options = options process_args = {'messageLogger': message_logger} self.auto = RemoteAutomation(self.device, options.remoteappname, self.remoteProfile, self.remoteLogFile, processArgs=process_args) self.environment = self.auto.environment self.remoteScreenshots = "/mnt/sdcard/Robotium-Screenshots" self.remoteMozLog = posixpath.join(options.remoteTestRoot, "mozlog") self.localLog = options.logFile self.localProfile = None self.certdbNew = True self.passed = 0 self.failed = 0 self.todo = 0 def startup(self): """ Second-stage initialization: One-time initialization which may require cleanup. """ # Despite our efforts to clean up servers started by this script, in practice # we still see infrequent cases where a process is orphaned and interferes # with future tests, typically because the old server is keeping the port in use. # Try to avoid those failures by checking for and killing servers before # trying to start new ones. self.killNamedProc('ssltunnel') self.killNamedProc('xpcshell') self.auto.deleteANRs() self.auto.deleteTombstones() procName = self.options.app.split('/')[-1] self.device.stop_application(procName) if self.device.process_exist(procName): self.log.warning("unable to kill %s before running tests!" % procName) self.device.rm(self.remoteScreenshots, force=True, recursive=True) self.device.rm(self.remoteMozLog, force=True, recursive=True) self.device.mkdir(self.remoteMozLog) logParent = posixpath.dirname(self.remoteLogFile) self.device.rm(logParent, force=True, recursive=True) self.device.mkdir(logParent) # Add Android version (SDK level) to mozinfo so that manifest entries # can be conditional on android_version. self.log.info( "Android sdk version '%s'; will use this to filter manifests" % str(self.device.version)) mozinfo.info['android_version'] = str(self.device.version) if self.options.robocopApk: self.device.install_app(self.options.robocopApk, replace=True) self.log.debug("Robocop APK %s installed" % self.options.robocopApk) # Display remote diagnostics; if running in mach, keep output terse. if self.options.log_mach is None: self.printDeviceInfo() self.setupLocalPaths() self.buildProfile() self.startServers(self.options, debuggerInfo=None) self.log.debug("Servers started") def cleanup(self): """ Cleanup at end of job run. """ self.log.debug("Cleaning up...") self.stopServers() self.device.stop_application(self.options.app.split('/')[-1]) uploadDir = os.environ.get('MOZ_UPLOAD_DIR', None) if uploadDir: self.log.debug( "Pulling any remote moz logs and screenshots to %s." % uploadDir) if self.device.is_dir(self.remoteMozLog): self.device.pull(self.remoteMozLog, uploadDir) if self.device.is_dir(self.remoteScreenshots): self.device.pull(self.remoteScreenshots, uploadDir) MochitestDesktop.cleanup(self, self.options) if self.localProfile: mozfile.remove(self.localProfile) self.device.rm(self.remoteProfile, force=True, recursive=True) self.device.rm(self.remoteProfileCopy, force=True, recursive=True) self.device.rm(self.remoteScreenshots, force=True, recursive=True) self.device.rm(self.remoteMozLog, force=True, recursive=True) self.device.rm(self.remoteConfigFile, force=True) self.device.rm(self.remoteLogFile, force=True) self.log.debug("Cleanup complete.") def findPath(self, paths, filename=None): for path in paths: p = path if filename: p = os.path.join(p, filename) if os.path.exists(self.getFullPath(p)): return path return None def makeLocalAutomation(self): localAutomation = Automation() localAutomation.IS_WIN32 = False localAutomation.IS_LINUX = False localAutomation.IS_MAC = False localAutomation.UNIXISH = False hostos = sys.platform if (hostos == 'mac' or hostos == 'darwin'): localAutomation.IS_MAC = True elif (hostos == 'linux' or hostos == 'linux2'): localAutomation.IS_LINUX = True localAutomation.UNIXISH = True elif (hostos == 'win32' or hostos == 'win64'): localAutomation.BIN_SUFFIX = ".exe" localAutomation.IS_WIN32 = True return localAutomation def setupLocalPaths(self): """ Setup xrePath and utilityPath and verify xpcshell. This is similar to switchToLocalPaths in runtestsremote.py. """ localAutomation = self.makeLocalAutomation() paths = [self.options.xrePath, localAutomation.DIST_BIN] self.options.xrePath = self.findPath(paths) if self.options.xrePath is None: self.log.error( "unable to find xulrunner path for %s, please specify with --xre-path" % os.name) sys.exit(1) self.log.debug("using xre path %s" % self.options.xrePath) xpcshell = "xpcshell" if (os.name == "nt"): xpcshell += ".exe" if self.options.utilityPath: paths = [self.options.utilityPath, self.options.xrePath] else: paths = [self.options.xrePath] self.options.utilityPath = self.findPath(paths, xpcshell) if self.options.utilityPath is None: self.log.error( "unable to find utility path for %s, please specify with --utility-path" % os.name) sys.exit(1) self.log.debug("using utility path %s" % self.options.utilityPath) xpcshell_path = os.path.join(self.options.utilityPath, xpcshell) if localAutomation.elf_arm(xpcshell_path): self.log.error('xpcshell at %s is an ARM binary; please use ' 'the --utility-path argument to specify the path ' 'to a desktop version.' % xpcshell_path) sys.exit(1) self.log.debug("xpcshell found at %s" % xpcshell_path) def buildProfile(self): """ Build a profile locally, keep it locally for use by servers and push a copy to the remote profile-copy directory. This is similar to buildProfile in runtestsremote.py. """ self.options.extraPrefs.append('browser.search.suggest.enabled=true') self.options.extraPrefs.append('browser.search.suggest.prompted=true') self.options.extraPrefs.append('layout.css.devPixelsPerPx=1.0') self.options.extraPrefs.append('browser.chrome.dynamictoolbar=false') self.options.extraPrefs.append('browser.snippets.enabled=false') self.options.extraPrefs.append('extensions.autoupdate.enabled=false') # Override the telemetry init delay for integration testing. self.options.extraPrefs.append('toolkit.telemetry.initDelay=1') self.options.extensionsToExclude.extend([ '*****@*****.**', ]) self.extraPrefs = parse_preferences(self.options.extraPrefs) if self.options.testingModulesDir: try: self.device.push(self.options.testingModulesDir, self.remoteModulesDir) self.device.chmod(self.remoteModulesDir, recursive=True, root=True) except Exception: self.log.error( "Automation Error: Unable to copy test modules to device.") raise savedTestingModulesDir = self.options.testingModulesDir self.options.testingModulesDir = self.remoteModulesDir else: savedTestingModulesDir = None manifest = MochitestDesktop.buildProfile(self, self.options) if savedTestingModulesDir: self.options.testingModulesDir = savedTestingModulesDir self.localProfile = self.options.profilePath self.log.debug("Profile created at %s" % self.localProfile) # some files are not needed for robocop; save time by not pushing os.remove(os.path.join(self.localProfile, 'userChrome.css')) try: self.device.push(self.localProfile, self.remoteProfileCopy) except Exception: self.log.error( "Automation Error: Unable to copy profile to device.") raise return manifest def setupRemoteProfile(self): """ Remove any remote profile and re-create it. """ self.log.debug("Updating remote profile at %s" % self.remoteProfile) self.device.rm(self.remoteProfile, force=True, recursive=True) self.device.cp(self.remoteProfileCopy, self.remoteProfile, recursive=True) def parseLocalLog(self): """ Read and parse the local log file, noting any failures. """ with open(self.localLog) as currentLog: data = currentLog.readlines() os.unlink(self.localLog) start_found = False end_found = False fail_found = False for line in data: try: message = json.loads(line) if not isinstance(message, dict) or 'action' not in message: continue except ValueError: continue if message['action'] == 'test_end': end_found = True start_found = False break if start_found and not end_found: if 'status' in message: if 'expected' in message: self.failed += 1 elif message['status'] == 'PASS': self.passed += 1 elif message['status'] == 'FAIL': self.todo += 1 if message['action'] == 'test_start': start_found = True if 'expected' in message: fail_found = True result = 0 if fail_found: result = 1 if not end_found: self.log.info( "PROCESS-CRASH | Automation Error: Missing end of test marker (process crashed?)" ) result = 1 return result def logTestSummary(self): """ Print a summary of all tests run to stdout, for treeherder parsing (logging via self.log does not work here). """ print("0 INFO TEST-START | Shutdown") print("1 INFO Passed: %s" % (self.passed)) print("2 INFO Failed: %s" % (self.failed)) print("3 INFO Todo: %s" % (self.todo)) print("4 INFO SimpleTest FINISHED") if self.failed > 0: return 1 return 0 def printDeviceInfo(self, printLogcat=False): """ Log remote device information and logcat (if requested). This is similar to printDeviceInfo in runtestsremote.py """ try: if printLogcat: logcat = self.device.get_logcat( filter_out_regexps=fennecLogcatFilters) for l in logcat: self.log.info(l.decode('utf-8', 'replace')) self.log.info("Device info:") devinfo = self.device.get_info() for category in devinfo: if type(devinfo[category]) is list: self.log.info(" %s:" % category) for item in devinfo[category]: self.log.info(" %s" % item) else: self.log.info(" %s: %s" % (category, devinfo[category])) self.log.info("Test root: %s" % self.device.test_root) except ADBTimeoutError: raise except Exception as e: self.log.warning("Error getting device information: %s" % str(e)) def setupRobotiumConfig(self, browserEnv): """ Create robotium.config and push it to the device. """ fHandle = tempfile.NamedTemporaryFile(suffix='.config', prefix='robotium-', dir=os.getcwd(), delete=False) fHandle.write("profile=%s\n" % self.remoteProfile) fHandle.write("logfile=%s\n" % self.remoteLogFile) fHandle.write("host=http://mochi.test:8888/tests\n") fHandle.write("rawhost=http://%s:%s/tests\n" % (self.options.remoteWebServer, self.options.httpPort)) if browserEnv: envstr = "" delim = "" for key, value in browserEnv.items(): try: value.index(',') self.log.error( "setupRobotiumConfig: browserEnv - Found a ',' " "in our value, unable to process value. key=%s,value=%s" % (key, value)) self.log.error("browserEnv=%s" % browserEnv) except ValueError: envstr += "%s%s=%s" % (delim, key, value) delim = "," fHandle.write("envvars=%s\n" % envstr) fHandle.close() self.device.rm(self.remoteConfigFile, force=True) self.device.push(fHandle.name, self.remoteConfigFile) os.unlink(fHandle.name) def buildBrowserEnv(self): """ Return an environment dictionary suitable for remote use. This is similar to buildBrowserEnv in runtestsremote.py. """ browserEnv = self.environment(xrePath=None, debugger=None) # remove desktop environment not used on device if "XPCOM_MEM_BLOAT_LOG" in browserEnv: del browserEnv["XPCOM_MEM_BLOAT_LOG"] browserEnv["MOZ_LOG_FILE"] = os.path.join(self.remoteMozLog, self.mozLogName) try: browserEnv.update( dict( parse_key_value(self.options.environment, context='--setenv'))) except KeyValueParseError as e: self.log.error(str(e)) return None return browserEnv def runSingleTest(self, test): """ Run the specified test. """ self.log.debug("Running test %s" % test['name']) self.mozLogName = "moz-%s.log" % test['name'] browserEnv = self.buildBrowserEnv() self.setupRobotiumConfig(browserEnv) self.setupRemoteProfile() self.options.app = "am" timeout = None testName = test['name'].split('/')[-1].split('.java')[0] if self.options.enable_coverage: remoteCoverageFile = posixpath.join( self.options.remoteTestRoot, 'robocop-coverage-%s.ec' % testName) coverageFile = os.path.join(self.options.coverage_output_dir, 'robocop-coverage-%s.ec' % testName) if self.options.autorun: # This launches a test (using "am instrument") and instructs # Fennec to /quit/ the browser (using Robocop:Quit) and to # /finish/ all opened activities. browserArgs = [ "instrument", ] if self.options.enable_coverage: browserArgs += [ "-e", "coverage", "true", "-e", "coverageFile", remoteCoverageFile, ] browserArgs += [ "-e", "quit_and_finish", "1", "-e", "deviceroot", self.device.test_root, "-e", "class", "org.mozilla.gecko.tests.%s" % testName, "org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner", ] else: # This does not launch a test at all. It launches an activity # that starts Fennec and then waits indefinitely, since cat # never returns. browserArgs = [ "start", "-n", "org.mozilla.roboexample.test/org.mozilla." "gecko.LaunchFennecWithConfigurationActivity", "&&", "cat" ] timeout = sys.maxint # Forever. self.log.info("") self.log.info( "Serving mochi.test Robocop root at http://%s:%s/tests/robocop/" % (self.options.remoteWebServer, self.options.httpPort)) self.log.info("") result = -1 log_result = -1 try: self.device.clear_logcat() if not timeout: timeout = self.options.timeout if not timeout: timeout = self.NO_OUTPUT_TIMEOUT result, _ = self.auto.runApp(None, browserEnv, "am", self.localProfile, browserArgs, timeout=timeout, symbolsPath=self.options.symbolsPath) self.log.debug("runApp completes with status %d" % result) if result != 0: self.log.error("runApp() exited with code %s" % result) if self.device.is_file(self.remoteLogFile): self.device.pull(self.remoteLogFile, self.localLog) self.device.rm(self.remoteLogFile) log_result = self.parseLocalLog() if result != 0 or log_result != 0: # Display remote diagnostics; if running in mach, keep output # terse. if self.options.log_mach is None: self.printDeviceInfo(printLogcat=True) if self.options.enable_coverage: if self.device.is_file(remoteCoverageFile): self.device.pull(remoteCoverageFile, coverageFile) self.device.rm(remoteCoverageFile) else: self.log.warning( "Code coverage output not found on remote device: %s" % remoteCoverageFile) except Exception: self.log.error( "Automation Error: Exception caught while running tests") traceback.print_exc() result = 1 self.log.debug("Test %s completes with status %d (log status %d)" % (test['name'], int(result), int(log_result))) return result def runTests(self): self.startup() if isinstance(self.options.manifestFile, TestManifest): mp = self.options.manifestFile else: mp = TestManifest(strict=False) mp.read("robocop.ini") filters = [] if self.options.totalChunks: filters.append( chunk_by_slice(self.options.thisChunk, self.options.totalChunks)) robocop_tests = mp.active_tests(exists=False, filters=filters, **mozinfo.info) if not self.options.autorun: # Force a single loop iteration. The iteration will start Fennec and # the httpd server, but not actually run a test. self.options.test_paths = [robocop_tests[0]['name']] active_tests = [] for test in robocop_tests: if self.options.test_paths and test[ 'name'] not in self.options.test_paths: continue if 'disabled' in test: self.log.info('TEST-INFO | skipping %s | %s' % (test['name'], test['disabled'])) continue active_tests.append(test) tests_by_manifest = defaultdict(list) for test in active_tests: tests_by_manifest[test['manifest']].append(test['name']) self.log.suite_start(tests_by_manifest) worstTestResult = None for test in active_tests: result = self.runSingleTest(test) if worstTestResult is None or worstTestResult == 0: worstTestResult = result if worstTestResult is None: self.log.warning( "No tests run. Did you pass an invalid TEST_PATH?") worstTestResult = 1 else: print "INFO | runtests.py | Test summary: start." logResult = self.logTestSummary() print "INFO | runtests.py | Test summary: end." if worstTestResult == 0: worstTestResult = logResult return worstTestResult