class RemoteReftest(RefTest): use_marionette = False resolver_cls = RemoteReftestResolver def __init__(self, options, scriptDir): RefTest.__init__(self, options.suite) self.run_by_manifest = False self.scriptDir = scriptDir self.localLogName = options.localLogName verbose = False if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug': verbose = True print "set verbose!" self.device = ADBAndroid(adb=options.adb_path, device=options.deviceSerial, test_root=options.remoteTestRoot, verbose=verbose) if options.remoteTestRoot is None: options.remoteTestRoot = posixpath.join(self.device.test_root, "reftest") options.remoteProfile = posixpath.join(options.remoteTestRoot, "profile") options.remoteLogFile = posixpath.join(options.remoteTestRoot, "reftest.log") options.logFile = options.remoteLogFile self.remoteProfile = options.remoteProfile self.remoteTestRoot = options.remoteTestRoot if not options.ignoreWindowSize: parts = self.device.get_info('screen')['screen'][0].split() width = int(parts[0].split(':')[1]) height = int(parts[1].split(':')[1]) if (width < 1366 or height < 1050): self.error("ERROR: Invalid screen resolution %sx%s, " "please adjust to 1366x1050 or higher" % (width, height)) self._populate_logger(options) self.outputHandler = OutputHandler(self.log, options.utilityPath, options.symbolsPath) # RemoteAutomation.py's 'messageLogger' is also used by mochitest. Mimic a mochitest # MessageLogger object to re-use this code path. self.outputHandler.write = self.outputHandler.__call__ self.automation = RemoteAutomation(self.device, options.app, self.remoteProfile, options.remoteLogFile, processArgs=None) self.automation._processArgs['messageLogger'] = self.outputHandler self.environment = self.automation.environment if self.automation.IS_DEBUG_BUILD: self.SERVER_STARTUP_TIMEOUT = 180 else: self.SERVER_STARTUP_TIMEOUT = 90 self.remoteCache = os.path.join(options.remoteTestRoot, "cache/") # Check that Firefox is installed expected = options.app.split('/')[-1] if not self.device.is_app_installed(expected): raise Exception("%s is not installed on this device" % expected) self.automation.deleteANRs() self.automation.deleteTombstones() self.device.clear_logcat() self.device.rm(self.remoteCache, force=True, recursive=True) procName = options.app.split('/')[-1] self.device.pkill(procName) if self.device.process_exist(procName): self.log.error("unable to kill %s before starting tests!" % procName) def findPath(self, paths, filename=None): for path in paths: p = path if filename: p = os.path.join(p, filename) if os.path.exists(self.getFullPath(p)): return path return None def startWebServer(self, options): """ Create the webserver on the host and start it up """ remoteXrePath = options.xrePath remoteUtilityPath = options.utilityPath localAutomation = Automation() localAutomation.IS_WIN32 = False localAutomation.IS_LINUX = False localAutomation.IS_MAC = False localAutomation.UNIXISH = False hostos = sys.platform if (hostos == 'mac' or hostos == 'darwin'): localAutomation.IS_MAC = True elif (hostos == 'linux' or hostos == 'linux2'): localAutomation.IS_LINUX = True localAutomation.UNIXISH = True elif (hostos == 'win32' or hostos == 'win64'): localAutomation.BIN_SUFFIX = ".exe" localAutomation.IS_WIN32 = True paths = [options.xrePath, localAutomation.DIST_BIN] options.xrePath = self.findPath(paths) if options.xrePath is None: print( "ERROR: unable to find xulrunner path for %s, " "please specify with --xre-path" % (os.name)) return 1 paths.append("bin") paths.append(os.path.join("..", "bin")) xpcshell = "xpcshell" if (os.name == "nt"): xpcshell += ".exe" if (options.utilityPath): paths.insert(0, options.utilityPath) options.utilityPath = self.findPath(paths, xpcshell) if options.utilityPath is None: print( "ERROR: unable to find utility path for %s, " "please specify with --utility-path" % (os.name)) return 1 options.serverProfilePath = tempfile.mkdtemp() self.server = ReftestServer(localAutomation, options, self.scriptDir) retVal = self.server.start() if retVal: return retVal retVal = self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT) if retVal: return retVal options.xrePath = remoteXrePath options.utilityPath = remoteUtilityPath return 0 def stopWebServer(self, options): self.server.stop() def killNamedProc(self, pname, orphans=True): """ Kill processes matching the given command name """ self.log.info("Checking for %s processes..." % pname) for proc in psutil.process_iter(): try: if proc.name() == pname: procd = proc.as_dict( attrs=['pid', 'ppid', 'name', 'username']) if proc.ppid() == 1 or not orphans: self.log.info("killing %s" % procd) try: os.kill(proc.pid, getattr(signal, "SIGKILL", signal.SIGTERM)) except Exception as e: self.log.info("Failed to kill process %d: %s" % (proc.pid, str(e))) else: self.log.info("NOT killing %s (not an orphan?)" % procd) except Exception: # may not be able to access process info for all processes continue def createReftestProfile(self, options, **kwargs): profile = RefTest.createReftestProfile(self, options, server=options.remoteWebServer, port=options.httpPort, **kwargs) profileDir = profile.profile prefs = {} prefs["app.update.url.android"] = "" prefs["browser.firstrun.show.localepicker"] = False prefs["reftest.remote"] = True prefs[ "datareporting.policy.dataSubmissionPolicyBypassAcceptance"] = True # move necko cache to a location that can be cleaned up prefs["browser.cache.disk.parent_directory"] = self.remoteCache prefs["layout.css.devPixelsPerPx"] = "1.0" # Because Fennec is a little wacky (see bug 1156817) we need to load the # reftest pages at 1.0 zoom, rather than zooming to fit the CSS viewport. prefs["apz.allow_zooming"] = False # Set the extra prefs. profile.set_preferences(prefs) try: self.device.push(profileDir, options.remoteProfile) self.device.chmod(options.remoteProfile, recursive=True) except Exception: print "Automation Error: Failed to copy profiledir to device" raise return profile def copyExtraFilesToProfile(self, options, profile): profileDir = profile.profile RefTest.copyExtraFilesToProfile(self, options, profile) if len(os.listdir(profileDir)) > 0: try: self.device.push(profileDir, options.remoteProfile) self.device.chmod(options.remoteProfile, recursive=True) except Exception: print "Automation Error: Failed to copy extra files to device" raise def printDeviceInfo(self, printLogcat=False): try: if printLogcat: logcat = self.device.get_logcat( filter_out_regexps=fennecLogcatFilters) print ''.join(logcat) print "Device info:" devinfo = self.device.get_info() for category in devinfo: if type(devinfo[category]) is list: print " %s:" % category for item in devinfo[category]: print " %s" % item else: print " %s: %s" % (category, devinfo[category]) print "Test root: %s" % self.device.test_root except Exception as e: print "WARNING: Error getting device information: %s" % str(e) def environment(self, **kwargs): return self.automation.environment(**kwargs) def buildBrowserEnv(self, options, profileDir): browserEnv = RefTest.buildBrowserEnv(self, options, profileDir) # remove desktop environment not used on device if "XPCOM_MEM_BLOAT_LOG" in browserEnv: del browserEnv["XPCOM_MEM_BLOAT_LOG"] return browserEnv def runApp(self, options, cmdargs=None, timeout=None, debuggerInfo=None, symbolsPath=None, valgrindPath=None, valgrindArgs=None, valgrindSuppFiles=None, **profileArgs): if cmdargs is None: cmdargs = [] if self.use_marionette: cmdargs.append('-marionette') binary = options.app profile = self.createReftestProfile(options, **profileArgs) # browser environment env = self.buildBrowserEnv(options, profile.profile) self.log.info("Running with e10s: {}".format(options.e10s)) status, self.lastTestSeen = self.automation.runApp( None, env, binary, profile.profile, cmdargs, utilityPath=options.utilityPath, xrePath=options.xrePath, debuggerInfo=debuggerInfo, symbolsPath=symbolsPath, timeout=timeout) self.cleanup(profile.profile) return status def cleanup(self, profileDir): self.device.rm(self.remoteTestRoot, force=True, recursive=True) self.device.rm(self.remoteProfile, force=True, recursive=True) self.device.rm(self.remoteCache, force=True, recursive=True) RefTest.cleanup(self, profileDir)
class Raptor(object): """Container class for Raptor""" def __init__(self, app, binary, run_local=False, obj_path=None): self.config = {} self.config['app'] = app self.config['binary'] = binary self.config['platform'] = mozinfo.os self.config['processor'] = mozinfo.processor self.config['run_local'] = run_local self.config['obj_path'] = obj_path self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv') self.log = get_default_logger(component='raptor-main') self.control_server = None self.playback = None self.benchmark = None # Create the profile; for geckoview we want a firefox profile type if self.config['app'] == 'geckoview': self.profile = create_profile('firefox') else: self.profile = create_profile(self.config['app']) # Merge in base profiles with open(os.path.join(self.profile_data_dir, 'profiles.json'), 'r') as fh: base_profiles = json.load(fh)['raptor'] for name in base_profiles: path = os.path.join(self.profile_data_dir, name) self.log.info("Merging profile: {}".format(path)) self.profile.merge(path) # create results holder self.results_handler = RaptorResultsHandler() # when testing desktop browsers we use mozrunner to start the browser; when # testing on android (i.e. geckoview) we use mozdevice to control the device app if self.config['app'] == "geckoview": # create the android device handler; it gets initiated and sets up adb etc self.log.info("creating android device handler using mozdevice") self.device = ADBAndroid(verbose=True) self.device.clear_logcat() else: # create the desktop browser runner self.log.info("creating browser runner using mozrunner") self.output_handler = OutputHandler() process_args = { 'processOutputLine': [self.output_handler], } runner_cls = runners[app] self.runner = runner_cls(binary, profile=self.profile, process_args=process_args) self.log.info("raptor config: %s" % str(self.config)) @property def profile_data_dir(self): if 'MOZ_DEVELOPER_REPO_DIR' in os.environ: return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'profiles') if build: return os.path.join(build.topsrcdir, 'testing', 'profiles') return os.path.join(here, 'profile_data') def start_control_server(self): self.control_server = RaptorControlServer(self.results_handler) self.control_server.start() # for android we must make the control server available to the device if self.config['app'] == "geckoview": self.log.info( "making the raptor control server port available to device") _tcp_port = "tcp:%s" % self.control_server.port self.device.create_socket_connection('reverse', _tcp_port, _tcp_port) def get_playback_config(self, test): self.config['playback_tool'] = test.get('playback') self.log.info("test uses playback tool: %s " % self.config['playback_tool']) self.config['playback_binary_manifest'] = test.get( 'playback_binary_manifest', None) _key = 'playback_binary_zip_%s' % self.config['platform'] self.config['playback_binary_zip'] = test.get(_key, None) self.config['playback_pageset_manifest'] = test.get( 'playback_pageset_manifest', None) _key = 'playback_pageset_zip_%s' % self.config['platform'] self.config['playback_pageset_zip'] = test.get(_key, None) self.config['playback_recordings'] = test.get('playback_recordings', None) self.config['python3_win_manifest'] = test.get('python3_win_manifest', None) def run_test(self, test, timeout=None): self.log.info("starting raptor test: %s" % test['name']) self.log.info("test settings: %s" % str(test)) self.log.info("raptor config: %s" % str(self.config)) # benchmark-type tests require the benchmark test to be served out if test.get('type') == "benchmark": self.benchmark = Benchmark(self.config, test) benchmark_port = int(self.benchmark.port) else: benchmark_port = 0 gen_test_config(self.config['app'], test['name'], self.control_server.port, benchmark_port) # for android we must make the benchmarks server available to the device if self.config['app'] == "geckoview": self.log.info( "making the raptor benchmarks server port available to device") _tcp_port = "tcp:%s" % benchmark_port self.device.create_socket_connection('reverse', _tcp_port, _tcp_port) # must intall raptor addon each time because we dynamically update some content raptor_webext = os.path.join(webext_dir, 'raptor') self.log.info("installing webext %s" % raptor_webext) self.profile.addons.install(raptor_webext) # add test specific preferences if test.get("preferences", None) is not None: if self.config['app'] == "firefox": self.profile.set_preferences(json.loads(test['preferences'])) else: self.log.info("preferences were configured for the test, \ but we do not install them on non Firefox browsers." ) # on firefox we can get an addon id; chrome addon actually is just cmd line arg if self.config['app'] in ["firefox", "geckoview"]: webext_id = self.profile.addons.addon_details(raptor_webext)['id'] # some tests require tools to playback the test pages if test.get('playback', None) is not None: self.get_playback_config(test) # startup the playback tool self.playback = get_playback(self.config) # for geckoview we must copy the profile onto the device and set perms if self.config['app'] == "geckoview": if not self.device.is_app_installed(self.config['binary']): raise Exception('%s is not installed' % self.config['binary']) self.log.info("copying firefox profile onto the android device") self.device_profile = "/sdcard/raptor-profile" if self.device.is_dir(self.device_profile): self.device.rm(self.device_profile, recursive=True) self.device.mkdir(self.device_profile) self.device.push(self.profile.profile, self.device_profile) self.log.info("setting permisions to profile dir on the device") self.device.chmod(self.device_profile, recursive=True) # now start the geckoview app self.log.info("starting %s" % self.config['app']) extra_args = [ "-profile", self.device_profile, "--es", "env0", "LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6" ] try: # make sure the geckoview app is not running before # attempting to start. self.device.stop_application(self.config['binary']) self.device.launch_activity(self.config['binary'], "GeckoViewActivity", extra_args=extra_args, url='about:blank', fail_if_running=False) except Exception: self.log.error("Exception launching %s" % self.config['binary']) raise self.control_server.device = self.device self.control_server.app_name = self.config['binary'] else: # now start the desktop browser self.log.info("starting %s" % self.config['app']) self.runner.start() proc = self.runner.process_handler self.output_handler.proc = proc self.control_server.browser_proc = proc # set our cs flag to indicate we are running the browser/app self.control_server._finished = False # convert to seconds and account for page cycles timeout = int(timeout / 1000) * int(test['page_cycles']) try: elapsed_time = 0 while not self.control_server._finished: time.sleep(1) elapsed_time += 1 if elapsed_time > (timeout) - 5: # stop 5 seconds early self.log.info( "application timed out after {} seconds".format( timeout)) self.control_server.wait_for_quit() break finally: if self.config['app'] != "geckoview": try: self.runner.check_for_crashes() except NotImplementedError: # not implemented for Chrome pass # TODO: if on geckoview is there some cleanup here i.e. check for crashes? if self.playback is not None: self.playback.stop() # remove the raptor webext; as it must be reloaded with each subtest anyway # applies to firefox only; chrome the addon is actually just cmd line arg if self.config['app'] in ["firefox", "geckoview"]: self.log.info("removing webext %s" % raptor_webext) self.profile.addons.remove_addon(webext_id) if self.config['app'] != "geckoview": if self.runner.is_running(): self.runner.stop() # TODO the geckoview app should have been shutdown by this point by the # control server, but we can double-check here to make sure def process_results(self): # when running locally output results in build/raptor.json; when running # in production output to a local.json to be turned into tc job artifact if self.config.get('run_local', False): if 'MOZ_DEVELOPER_REPO_DIR' in os.environ: raptor_json_path = os.path.join( os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'mozharness', 'build', 'raptor.json') else: raptor_json_path = os.path.join(here, 'raptor.json') else: raptor_json_path = os.path.join(os.getcwd(), 'local.json') self.config['raptor_json_path'] = raptor_json_path return self.results_handler.summarize_and_output(self.config) def clean_up(self): self.control_server.stop() if self.config['app'] != "geckoview": self.runner.stop() elif self.config['app'] == 'geckoview': self.log.info('removing reverse socket connections') self.device.remove_socket_connections('reverse') else: pass self.log.info("finished")
class MochiRemote(MochitestDesktop): localProfile = None logMessages = [] def __init__(self, options): MochitestDesktop.__init__(self, options.flavor, vars(options)) verbose = False if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug': verbose = True if hasattr(options, 'log'): delattr(options, 'log') self.certdbNew = True self.chromePushed = False self.mozLogName = "moz.log" self.device = ADBAndroid(adb=options.adbPath, device=options.deviceSerial, test_root=options.remoteTestRoot, verbose=verbose) if options.remoteTestRoot is None: options.remoteTestRoot = self.device.test_root options.dumpOutputDirectory = options.remoteTestRoot self.remoteLogFile = posixpath.join(options.remoteTestRoot, "logs", "mochitest.log") logParent = posixpath.dirname(self.remoteLogFile) self.device.rm(logParent, force=True, recursive=True) self.device.mkdir(logParent) self.remoteProfile = posixpath.join(options.remoteTestRoot, "profile/") self.device.rm(self.remoteProfile, force=True, recursive=True) self.counts = dict() self.message_logger = MessageLogger(logger=None) self.message_logger.logger = self.log process_args = { 'messageLogger': self.message_logger, 'counts': self.counts } self.automation = RemoteAutomation(self.device, options.remoteappname, self.remoteProfile, self.remoteLogFile, processArgs=process_args) self.environment = self.automation.environment # Check that Firefox is installed expected = options.app.split('/')[-1] if not self.device.is_app_installed(expected): raise Exception("%s is not installed on this device" % expected) self.automation.deleteANRs() self.automation.deleteTombstones() self.device.clear_logcat() self.remoteModulesDir = posixpath.join(options.remoteTestRoot, "modules/") self.remoteCache = posixpath.join(options.remoteTestRoot, "cache/") self.device.rm(self.remoteCache, force=True, recursive=True) # move necko cache to a location that can be cleaned up options.extraPrefs += [ "browser.cache.disk.parent_directory=%s" % self.remoteCache ] self.remoteMozLog = posixpath.join(options.remoteTestRoot, "mozlog") self.device.rm(self.remoteMozLog, force=True, recursive=True) self.device.mkdir(self.remoteMozLog) self.remoteChromeTestDir = posixpath.join(options.remoteTestRoot, "chrome") self.device.rm(self.remoteChromeTestDir, force=True, recursive=True) self.device.mkdir(self.remoteChromeTestDir) procName = options.app.split('/')[-1] self.device.stop_application(procName) if self.device.process_exist(procName): self.log.warning("unable to kill %s before running tests!" % procName) # Add Android version (SDK level) to mozinfo so that manifest entries # can be conditional on android_version. self.log.info( "Android sdk version '%s'; will use this to filter manifests" % str(self.device.version)) mozinfo.info['android_version'] = str(self.device.version) def cleanup(self, options, final=False): if final: self.device.rm(self.remoteChromeTestDir, force=True, recursive=True) self.chromePushed = False uploadDir = os.environ.get('MOZ_UPLOAD_DIR', None) if uploadDir and self.device.is_dir(self.remoteMozLog): self.device.pull(self.remoteMozLog, uploadDir) self.device.rm(self.remoteLogFile, force=True) self.device.rm(self.remoteProfile, force=True, recursive=True) self.device.rm(self.remoteCache, force=True, recursive=True) MochitestDesktop.cleanup(self, options, final) self.localProfile = None def findPath(self, paths, filename=None): for path in paths: p = path if filename: p = os.path.join(p, filename) if os.path.exists(self.getFullPath(p)): return path return None def makeLocalAutomation(self): localAutomation = Automation() localAutomation.IS_WIN32 = False localAutomation.IS_LINUX = False localAutomation.IS_MAC = False localAutomation.UNIXISH = False hostos = sys.platform if (hostos == 'mac' or hostos == 'darwin'): localAutomation.IS_MAC = True elif (hostos == 'linux' or hostos == 'linux2'): localAutomation.IS_LINUX = True localAutomation.UNIXISH = True elif (hostos == 'win32' or hostos == 'win64'): localAutomation.BIN_SUFFIX = ".exe" localAutomation.IS_WIN32 = True return localAutomation # This seems kludgy, but this class uses paths from the remote host in the # options, except when calling up to the base class, which doesn't # understand the distinction. This switches out the remote values for local # ones that the base class understands. This is necessary for the web # server, SSL tunnel and profile building functions. def switchToLocalPaths(self, options): """ Set local paths in the options, return a function that will restore remote values """ remoteXrePath = options.xrePath remoteProfilePath = options.profilePath remoteUtilityPath = options.utilityPath localAutomation = self.makeLocalAutomation() paths = [ options.xrePath, localAutomation.DIST_BIN, ] options.xrePath = self.findPath(paths) if options.xrePath is None: self.log.error( "unable to find xulrunner path for %s, please specify with --xre-path" % os.name) sys.exit(1) xpcshell = "xpcshell" if (os.name == "nt"): xpcshell += ".exe" if options.utilityPath: paths = [options.utilityPath, options.xrePath] else: paths = [options.xrePath] options.utilityPath = self.findPath(paths, xpcshell) if options.utilityPath is None: self.log.error( "unable to find utility path for %s, please specify with --utility-path" % os.name) sys.exit(1) xpcshell_path = os.path.join(options.utilityPath, xpcshell) if localAutomation.elf_arm(xpcshell_path): self.log.error('xpcshell at %s is an ARM binary; please use ' 'the --utility-path argument to specify the path ' 'to a desktop version.' % xpcshell_path) sys.exit(1) if self.localProfile: options.profilePath = self.localProfile else: options.profilePath = None def fixup(): options.xrePath = remoteXrePath options.utilityPath = remoteUtilityPath options.profilePath = remoteProfilePath return fixup def startServers(self, options, debuggerInfo): """ Create the servers on the host and start them up """ restoreRemotePaths = self.switchToLocalPaths(options) # ignoreSSLTunnelExts is a workaround for bug 1109310 MochitestDesktop.startServers(self, options, debuggerInfo, ignoreSSLTunnelExts=True) restoreRemotePaths() def buildProfile(self, options): restoreRemotePaths = self.switchToLocalPaths(options) if options.testingModulesDir: try: self.device.push(options.testingModulesDir, self.remoteModulesDir) self.device.chmod(self.remoteModulesDir, recursive=True) except Exception: self.log.error( "Automation Error: Unable to copy test modules to device.") raise savedTestingModulesDir = options.testingModulesDir options.testingModulesDir = self.remoteModulesDir else: savedTestingModulesDir = None manifest = MochitestDesktop.buildProfile(self, options) if savedTestingModulesDir: options.testingModulesDir = savedTestingModulesDir self.localProfile = options.profilePath restoreRemotePaths() options.profilePath = self.remoteProfile return manifest def buildURLOptions(self, options, env): saveLogFile = options.logFile options.logFile = self.remoteLogFile options.profilePath = self.localProfile env["MOZ_HIDE_RESULTS_TABLE"] = "1" retVal = MochitestDesktop.buildURLOptions(self, options, env) # we really need testConfig.js (for browser chrome) try: self.device.push(options.profilePath, self.remoteProfile) self.device.chmod(self.remoteProfile, recursive=True) except Exception: self.log.error( "Automation Error: Unable to copy profile to device.") raise options.profilePath = self.remoteProfile options.logFile = saveLogFile return retVal def getChromeTestDir(self, options): local = super(MochiRemote, self).getChromeTestDir(options) remote = self.remoteChromeTestDir if options.flavor == 'chrome' and not self.chromePushed: self.log.info("pushing %s to %s on device..." % (local, remote)) local = os.path.join(local, "chrome") self.device.push(local, remote) self.chromePushed = True return remote def getLogFilePath(self, logFile): return logFile def printDeviceInfo(self, printLogcat=False): try: if printLogcat: logcat = self.device.get_logcat( filter_out_regexps=fennecLogcatFilters) self.log.info('\n' + ''.join(logcat).decode('utf-8', 'replace')) self.log.info("Device info:") devinfo = self.device.get_info() for category in devinfo: if type(devinfo[category]) is list: self.log.info(" %s:" % category) for item in devinfo[category]: self.log.info(" %s" % item) else: self.log.info(" %s: %s" % (category, devinfo[category])) self.log.info("Test root: %s" % self.device.test_root) except Exception as e: self.log.warning("Error getting device information: %s" % str(e)) def getGMPPluginPath(self, options): # TODO: bug 1149374 return None def buildBrowserEnv(self, options, debugger=False): browserEnv = MochitestDesktop.buildBrowserEnv(self, options, debugger=debugger) # remove desktop environment not used on device if "XPCOM_MEM_BLOAT_LOG" in browserEnv: del browserEnv["XPCOM_MEM_BLOAT_LOG"] # override mozLogs to avoid processing in MochitestDesktop base class self.mozLogs = None browserEnv["MOZ_LOG_FILE"] = os.path.join(self.remoteMozLog, self.mozLogName) if options.dmd: browserEnv['DMD'] = '1' return browserEnv def runApp(self, *args, **kwargs): """front-end automation.py's `runApp` functionality until FennecRunner is written""" # automation.py/remoteautomation `runApp` takes the profile path, # whereas runtest.py's `runApp` takes a mozprofile object. if 'profileDir' not in kwargs and 'profile' in kwargs: kwargs['profileDir'] = kwargs.pop('profile').profile # remove args not supported by automation.py kwargs.pop('marionette_args', None) ret, _ = self.automation.runApp(*args, **kwargs) self.countpass += self.counts['pass'] self.countfail += self.counts['fail'] self.counttodo += self.counts['todo'] return ret, None
class RobocopTestRunner(MochitestDesktop): """ A test harness for Robocop. Robocop tests are UI tests for Firefox for Android, based on the Robotium test framework. This harness leverages some functionality from mochitest, for convenience. """ # Some robocop tests run for >60 seconds without generating any output. NO_OUTPUT_TIMEOUT = 180 def __init__(self, options, message_logger): """ Simple one-time initialization. """ MochitestDesktop.__init__(self, options.flavor, vars(options)) verbose = False if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug': verbose = True self.device = ADBAndroid(adb=options.adbPath, device=options.deviceSerial, test_root=options.remoteTestRoot, verbose=verbose) # Check that Firefox is installed expected = options.app.split('/')[-1] if not self.device.is_app_installed(expected): raise Exception("%s is not installed on this device" % expected) options.logFile = "robocop.log" if options.remoteTestRoot is None: options.remoteTestRoot = self.device.test_root self.remoteProfile = posixpath.join(options.remoteTestRoot, "profile") self.remoteProfileCopy = posixpath.join(options.remoteTestRoot, "profile-copy") self.remoteConfigFile = posixpath.join(options.remoteTestRoot, "robotium.config") self.remoteLogFile = posixpath.join(options.remoteTestRoot, "logs", "robocop.log") self.options = options process_args = {'messageLogger': message_logger} self.auto = RemoteAutomation(self.device, options.remoteappname, self.remoteProfile, self.remoteLogFile, processArgs=process_args) self.environment = self.auto.environment self.remoteScreenshots = "/mnt/sdcard/Robotium-Screenshots" self.remoteMozLog = posixpath.join(options.remoteTestRoot, "mozlog") self.localLog = options.logFile self.localProfile = None self.certdbNew = True self.passed = 0 self.failed = 0 self.todo = 0 def startup(self): """ Second-stage initialization: One-time initialization which may require cleanup. """ # Despite our efforts to clean up servers started by this script, in practice # we still see infrequent cases where a process is orphaned and interferes # with future tests, typically because the old server is keeping the port in use. # Try to avoid those failures by checking for and killing servers before # trying to start new ones. self.killNamedProc('ssltunnel') self.killNamedProc('xpcshell') self.auto.deleteANRs() self.auto.deleteTombstones() procName = self.options.app.split('/')[-1] self.device.stop_application(procName) if self.device.process_exist(procName): self.log.warning("unable to kill %s before running tests!" % procName) self.device.rm(self.remoteScreenshots, force=True, recursive=True) self.device.rm(self.remoteMozLog, force=True, recursive=True) self.device.mkdir(self.remoteMozLog) logParent = posixpath.dirname(self.remoteLogFile) self.device.rm(logParent, force=True, recursive=True) self.device.mkdir(logParent) # Add Android version (SDK level) to mozinfo so that manifest entries # can be conditional on android_version. self.log.info( "Android sdk version '%s'; will use this to filter manifests" % str(self.device.version)) mozinfo.info['android_version'] = str(self.device.version) if self.options.robocopApk: self.device.install_app(self.options.robocopApk, replace=True) self.log.debug("Robocop APK %s installed" % self.options.robocopApk) # Display remote diagnostics; if running in mach, keep output terse. if self.options.log_mach is None: self.printDeviceInfo() self.setupLocalPaths() self.buildProfile() # ignoreSSLTunnelExts is a workaround for bug 1109310 self.startServers(self.options, debuggerInfo=None, ignoreSSLTunnelExts=True) self.log.debug("Servers started") def cleanup(self): """ Cleanup at end of job run. """ self.log.debug("Cleaning up...") self.stopServers() self.device.stop_application(self.options.app.split('/')[-1]) uploadDir = os.environ.get('MOZ_UPLOAD_DIR', None) if uploadDir: self.log.debug( "Pulling any remote moz logs and screenshots to %s." % uploadDir) if self.device.is_dir(self.remoteMozLog): self.device.pull(self.remoteMozLog, uploadDir) if self.device.is_dir(self.remoteScreenshots): self.device.pull(self.remoteScreenshots, uploadDir) MochitestDesktop.cleanup(self, self.options) if self.localProfile: mozfile.remove(self.localProfile) self.device.rm(self.remoteProfile, force=True, recursive=True) self.device.rm(self.remoteProfileCopy, force=True, recursive=True) self.device.rm(self.remoteScreenshots, force=True, recursive=True) self.device.rm(self.remoteMozLog, force=True, recursive=True) self.device.rm(self.remoteConfigFile, force=True) self.device.rm(self.remoteLogFile, force=True) self.log.debug("Cleanup complete.") def findPath(self, paths, filename=None): for path in paths: p = path if filename: p = os.path.join(p, filename) if os.path.exists(self.getFullPath(p)): return path return None def makeLocalAutomation(self): localAutomation = Automation() localAutomation.IS_WIN32 = False localAutomation.IS_LINUX = False localAutomation.IS_MAC = False localAutomation.UNIXISH = False hostos = sys.platform if (hostos == 'mac' or hostos == 'darwin'): localAutomation.IS_MAC = True elif (hostos == 'linux' or hostos == 'linux2'): localAutomation.IS_LINUX = True localAutomation.UNIXISH = True elif (hostos == 'win32' or hostos == 'win64'): localAutomation.BIN_SUFFIX = ".exe" localAutomation.IS_WIN32 = True return localAutomation def setupLocalPaths(self): """ Setup xrePath and utilityPath and verify xpcshell. This is similar to switchToLocalPaths in runtestsremote.py. """ localAutomation = self.makeLocalAutomation() paths = [self.options.xrePath, localAutomation.DIST_BIN] self.options.xrePath = self.findPath(paths) if self.options.xrePath is None: self.log.error( "unable to find xulrunner path for %s, please specify with --xre-path" % os.name) sys.exit(1) self.log.debug("using xre path %s" % self.options.xrePath) xpcshell = "xpcshell" if (os.name == "nt"): xpcshell += ".exe" if self.options.utilityPath: paths = [self.options.utilityPath, self.options.xrePath] else: paths = [self.options.xrePath] self.options.utilityPath = self.findPath(paths, xpcshell) if self.options.utilityPath is None: self.log.error( "unable to find utility path for %s, please specify with --utility-path" % os.name) sys.exit(1) self.log.debug("using utility path %s" % self.options.utilityPath) xpcshell_path = os.path.join(self.options.utilityPath, xpcshell) if localAutomation.elf_arm(xpcshell_path): self.log.error('xpcshell at %s is an ARM binary; please use ' 'the --utility-path argument to specify the path ' 'to a desktop version.' % xpcshell_path) sys.exit(1) self.log.debug("xpcshell found at %s" % xpcshell_path) def buildProfile(self): """ Build a profile locally, keep it locally for use by servers and push a copy to the remote profile-copy directory. This is similar to buildProfile in runtestsremote.py. """ self.options.extraPrefs.append('browser.search.suggest.enabled=true') self.options.extraPrefs.append('browser.search.suggest.prompted=true') self.options.extraPrefs.append('layout.css.devPixelsPerPx=1.0') self.options.extraPrefs.append('browser.chrome.dynamictoolbar=false') self.options.extraPrefs.append('browser.snippets.enabled=false') self.options.extraPrefs.append('extensions.autoupdate.enabled=false') # Override the telemetry init delay for integration testing. self.options.extraPrefs.append('toolkit.telemetry.initDelay=1') self.options.extensionsToExclude.extend([ '*****@*****.**', '*****@*****.**', '*****@*****.**', ]) manifest = MochitestDesktop.buildProfile(self, self.options) self.localProfile = self.options.profilePath self.log.debug("Profile created at %s" % self.localProfile) # some files are not needed for robocop; save time by not pushing os.remove(os.path.join(self.localProfile, 'userChrome.css')) try: self.device.push(self.localProfile, self.remoteProfileCopy) except Exception: self.log.error( "Automation Error: Unable to copy profile to device.") raise return manifest def setupRemoteProfile(self): """ Remove any remote profile and re-create it. """ self.log.debug("Updating remote profile at %s" % self.remoteProfile) self.device.rm(self.remoteProfile, force=True, recursive=True) self.device.cp(self.remoteProfileCopy, self.remoteProfile, recursive=True) def parseLocalLog(self): """ Read and parse the local log file, noting any failures. """ with open(self.localLog) as currentLog: data = currentLog.readlines() os.unlink(self.localLog) start_found = False end_found = False fail_found = False for line in data: try: message = json.loads(line) if not isinstance(message, dict) or 'action' not in message: continue except ValueError: continue if message['action'] == 'test_end': end_found = True start_found = False break if start_found and not end_found: if 'status' in message: if 'expected' in message: self.failed += 1 elif message['status'] == 'PASS': self.passed += 1 elif message['status'] == 'FAIL': self.todo += 1 if message['action'] == 'test_start': start_found = True if 'expected' in message: fail_found = True result = 0 if fail_found: result = 1 if not end_found: self.log.info( "PROCESS-CRASH | Automation Error: Missing end of test marker (process crashed?)" ) result = 1 return result def logTestSummary(self): """ Print a summary of all tests run to stdout, for treeherder parsing (logging via self.log does not work here). """ print("0 INFO TEST-START | Shutdown") print("1 INFO Passed: %s" % (self.passed)) print("2 INFO Failed: %s" % (self.failed)) print("3 INFO Todo: %s" % (self.todo)) print("4 INFO SimpleTest FINISHED") if self.failed > 0: return 1 return 0 def printDeviceInfo(self, printLogcat=False): """ Log remote device information and logcat (if requested). This is similar to printDeviceInfo in runtestsremote.py """ try: if printLogcat: logcat = self.device.get_logcat( filter_out_regexps=fennecLogcatFilters) self.log.info('\n' + ''.join(logcat).decode('utf-8', 'replace')) self.log.info("Device info:") devinfo = self.device.get_info() for category in devinfo: if type(devinfo[category]) is list: self.log.info(" %s:" % category) for item in devinfo[category]: self.log.info(" %s" % item) else: self.log.info(" %s: %s" % (category, devinfo[category])) self.log.info("Test root: %s" % self.device.test_root) except Exception as e: self.log.warning("Error getting device information: %s" % str(e)) def setupRobotiumConfig(self, browserEnv): """ Create robotium.config and push it to the device. """ fHandle = tempfile.NamedTemporaryFile(suffix='.config', prefix='robotium-', dir=os.getcwd(), delete=False) fHandle.write("profile=%s\n" % self.remoteProfile) fHandle.write("logfile=%s\n" % self.remoteLogFile) fHandle.write("host=http://mochi.test:8888/tests\n") fHandle.write("rawhost=http://%s:%s/tests\n" % (self.options.remoteWebServer, self.options.httpPort)) if browserEnv: envstr = "" delim = "" for key, value in browserEnv.items(): try: value.index(',') self.log.error( "setupRobotiumConfig: browserEnv - Found a ',' " "in our value, unable to process value. key=%s,value=%s" % (key, value)) self.log.error("browserEnv=%s" % browserEnv) except ValueError: envstr += "%s%s=%s" % (delim, key, value) delim = "," fHandle.write("envvars=%s\n" % envstr) fHandle.close() self.device.rm(self.remoteConfigFile, force=True) self.device.push(fHandle.name, self.remoteConfigFile) os.unlink(fHandle.name) def buildBrowserEnv(self): """ Return an environment dictionary suitable for remote use. This is similar to buildBrowserEnv in runtestsremote.py. """ browserEnv = self.environment(xrePath=None, debugger=None) # remove desktop environment not used on device if "XPCOM_MEM_BLOAT_LOG" in browserEnv: del browserEnv["XPCOM_MEM_BLOAT_LOG"] browserEnv["MOZ_LOG_FILE"] = os.path.join(self.remoteMozLog, self.mozLogName) try: browserEnv.update( dict( parseKeyValue(self.options.environment, context='--setenv'))) except KeyValueParseError as e: self.log.error(str(e)) return None return browserEnv def runSingleTest(self, test): """ Run the specified test. """ self.log.debug("Running test %s" % test['name']) self.mozLogName = "moz-%s.log" % test['name'] browserEnv = self.buildBrowserEnv() self.setupRobotiumConfig(browserEnv) self.setupRemoteProfile() self.options.app = "am" timeout = None if self.options.autorun: # This launches a test (using "am instrument") and instructs # Fennec to /quit/ the browser (using Robocop:Quit) and to # /finish/ all opened activities. browserArgs = [ "instrument", "-e", "quit_and_finish", "1", "-e", "deviceroot", self.device.test_root, "-e", "class", "org.mozilla.gecko.tests.%s" % test['name'].split('/')[-1].split('.java')[0], "org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner" ] else: # This does not launch a test at all. It launches an activity # that starts Fennec and then waits indefinitely, since cat # never returns. browserArgs = [ "start", "-n", "org.mozilla.roboexample.test/org.mozilla." "gecko.LaunchFennecWithConfigurationActivity", "&&", "cat" ] timeout = sys.maxint # Forever. self.log.info("") self.log.info( "Serving mochi.test Robocop root at http://%s:%s/tests/robocop/" % (self.options.remoteWebServer, self.options.httpPort)) self.log.info("") result = -1 log_result = -1 try: self.device.clear_logcat() if not timeout: timeout = self.options.timeout if not timeout: timeout = self.NO_OUTPUT_TIMEOUT result, _ = self.auto.runApp(None, browserEnv, "am", self.localProfile, browserArgs, timeout=timeout, symbolsPath=self.options.symbolsPath) self.log.debug("runApp completes with status %d" % result) if result != 0: self.log.error("runApp() exited with code %s" % result) if self.device.is_file(self.remoteLogFile): self.device.pull(self.remoteLogFile, self.localLog) self.device.rm(self.remoteLogFile) log_result = self.parseLocalLog() if result != 0 or log_result != 0: # Display remote diagnostics; if running in mach, keep output # terse. if self.options.log_mach is None: self.printDeviceInfo(printLogcat=True) except Exception: self.log.error( "Automation Error: Exception caught while running tests") traceback.print_exc() result = 1 self.log.debug("Test %s completes with status %d (log status %d)" % (test['name'], int(result), int(log_result))) return result def runTests(self): self.startup() if isinstance(self.options.manifestFile, TestManifest): mp = self.options.manifestFile else: mp = TestManifest(strict=False) mp.read("robocop.ini") filters = [] if self.options.totalChunks: filters.append( chunk_by_slice(self.options.thisChunk, self.options.totalChunks)) robocop_tests = mp.active_tests(exists=False, filters=filters, **mozinfo.info) if not self.options.autorun: # Force a single loop iteration. The iteration will start Fennec and # the httpd server, but not actually run a test. self.options.test_paths = [robocop_tests[0]['name']] active_tests = [] for test in robocop_tests: if self.options.test_paths and test[ 'name'] not in self.options.test_paths: continue if 'disabled' in test: self.log.info('TEST-INFO | skipping %s | %s' % (test['name'], test['disabled'])) continue active_tests.append(test) tests_by_manifest = defaultdict(list) for test in active_tests: tests_by_manifest[test['manifest']].append(test['name']) self.log.suite_start(tests_by_manifest) worstTestResult = None for test in active_tests: result = self.runSingleTest(test) if worstTestResult is None or worstTestResult == 0: worstTestResult = result if worstTestResult is None: self.log.warning( "No tests run. Did you pass an invalid TEST_PATH?") worstTestResult = 1 else: print "INFO | runtests.py | Test summary: start." logResult = self.logTestSummary() print "INFO | runtests.py | Test summary: end." if worstTestResult == 0: worstTestResult = logResult return worstTestResult
class Raptor(object): """Container class for Raptor""" def __init__(self, app, binary, run_local=False, obj_path=None, gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None, symbols_path=None, host=None, is_release_build=False, debug_mode=False): self.config = {} self.config['app'] = app self.config['binary'] = binary self.config['platform'] = mozinfo.os self.config['processor'] = mozinfo.processor self.config['run_local'] = run_local self.config['obj_path'] = obj_path self.config['gecko_profile'] = gecko_profile self.config['gecko_profile_interval'] = gecko_profile_interval self.config['gecko_profile_entries'] = gecko_profile_entries self.config['symbols_path'] = symbols_path self.config['host'] = host self.config['is_release_build'] = is_release_build self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv') self.log = get_default_logger(component='raptor-main') self.control_server = None self.playback = None self.benchmark = None self.gecko_profiler = None self.post_startup_delay = 30000 self.device = None # debug mode is currently only supported when running locally self.debug_mode = debug_mode if self.config['run_local'] else False # if running debug-mode reduce the pause after browser startup if self.debug_mode: self.post_startup_delay = 3000 self.log.info( "debug-mode enabled, reducing post-browser startup pause to %d ms" % self.post_startup_delay) # Create the profile; for geckoview we want a firefox profile type if self.config['app'] == 'geckoview': self.profile = create_profile('firefox') else: self.profile = create_profile(self.config['app']) # Merge in base profiles with open(os.path.join(self.profile_data_dir, 'profiles.json'), 'r') as fh: base_profiles = json.load(fh)['raptor'] for name in base_profiles: path = os.path.join(self.profile_data_dir, name) self.log.info("Merging profile: {}".format(path)) self.profile.merge(path) # add profile dir to our config self.config['local_profile_dir'] = self.profile.profile # create results holder self.results_handler = RaptorResultsHandler() # when testing desktop browsers we use mozrunner to start the browser; when # testing on android (i.e. geckoview) we use mozdevice to control the device app if self.config['app'] == "geckoview": # create the android device handler; it gets initiated and sets up adb etc self.log.info("creating android device handler using mozdevice") self.device = ADBAndroid(verbose=True) self.device.clear_logcat() else: # create the desktop browser runner self.log.info("creating browser runner using mozrunner") self.output_handler = OutputHandler() process_args = { 'processOutputLine': [self.output_handler], } runner_cls = runners[app] self.runner = runner_cls(binary, profile=self.profile, process_args=process_args) self.log.info("raptor config: %s" % str(self.config)) @property def profile_data_dir(self): if 'MOZ_DEVELOPER_REPO_DIR' in os.environ: return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'profiles') if build: return os.path.join(build.topsrcdir, 'testing', 'profiles') return os.path.join(here, 'profile_data') def start_control_server(self): self.control_server = RaptorControlServer(self.results_handler, self.debug_mode) self.control_server.start() # for android we must make the control server available to the device if self.config['app'] == "geckoview" and self.config['host'] in ( 'localhost', '127.0.0.1'): self.log.info( "making the raptor control server port available to device") _tcp_port = "tcp:%s" % self.control_server.port self.device.create_socket_connection('reverse', _tcp_port, _tcp_port) def get_playback_config(self, test): self.config['playback_tool'] = test.get('playback') self.log.info("test uses playback tool: %s " % self.config['playback_tool']) self.config['playback_binary_manifest'] = test.get( 'playback_binary_manifest', None) _key = 'playback_binary_zip_%s' % self.config['platform'] self.config['playback_binary_zip'] = test.get(_key, None) self.config['playback_pageset_manifest'] = test.get( 'playback_pageset_manifest', None) _key = 'playback_pageset_zip_%s' % self.config['platform'] self.config['playback_pageset_zip'] = test.get(_key, None) self.config['playback_recordings'] = test.get('playback_recordings', None) self.config['python3_win_manifest'] = test.get('python3_win_manifest', None) def run_test(self, test, timeout=None): self.log.info("starting raptor test: %s" % test['name']) self.log.info("test settings: %s" % str(test)) self.log.info("raptor config: %s" % str(self.config)) # benchmark-type tests require the benchmark test to be served out if test.get('type') == "benchmark": self.benchmark = Benchmark(self.config, test) benchmark_port = int(self.benchmark.port) # for android we must make the benchmarks server available to the device if self.config['app'] == "geckoview" and self.config['host'] \ in ('localhost', '127.0.0.1'): self.log.info( "making the raptor benchmarks server port available to device" ) _tcp_port = "tcp:%s" % benchmark_port self.device.create_socket_connection('reverse', _tcp_port, _tcp_port) else: benchmark_port = 0 gen_test_config(self.config['app'], test['name'], self.control_server.port, self.post_startup_delay, host=self.config['host'], b_port=benchmark_port, debug_mode=1 if self.debug_mode else 0) # must intall raptor addon each time because we dynamically update some content # note: for chrome the addon is just a list of paths that ultimately are added # to the chromium command line '--load-extension' argument raptor_webext = os.path.join(webext_dir, 'raptor') self.log.info("installing webext %s" % raptor_webext) self.profile.addons.install(raptor_webext) # add test specific preferences if test.get("preferences", None) is not None: if self.config['app'] == "firefox": self.profile.set_preferences(json.loads(test['preferences'])) else: self.log.info("preferences were configured for the test, \ but we do not install them on non Firefox browsers." ) # on firefox we can get an addon id; chrome addon actually is just cmd line arg if self.config['app'] in ["firefox", "geckoview"]: webext_id = self.profile.addons.addon_details(raptor_webext)['id'] # for android/geckoview, create a top-level raptor folder on the device # sdcard; if it already exists remove it so we start fresh each time if self.config['app'] == "geckoview": self.device_raptor_dir = "/sdcard/raptor" self.config['device_raptor_dir'] = self.device_raptor_dir if self.device.is_dir(self.device_raptor_dir): self.log.info("deleting existing device raptor dir: %s" % self.device_raptor_dir) self.device.rm(self.device_raptor_dir, recursive=True) self.log.info("creating raptor folder on sdcard: %s" % self.device_raptor_dir) self.device.mkdir(self.device_raptor_dir) self.device.chmod(self.device_raptor_dir, recursive=True) # some tests require tools to playback the test pages if test.get('playback', None) is not None: self.get_playback_config(test) # startup the playback tool self.playback = get_playback(self.config, self.device) # for android we must make the playback server available to the device if self.config['app'] == "geckoview" and self.config['host'] \ in ('localhost', '127.0.0.1'): self.log.info( "making the raptor playback server port available to device" ) _tcp_port = "tcp:8080" self.device.create_socket_connection('reverse', _tcp_port, _tcp_port) if self.config['app'] in ("geckoview", "firefox") and \ self.config['host'] not in ('localhost', '127.0.0.1'): # Must delete the proxy settings from the profile if running # the test with a host different from localhost. userjspath = os.path.join(self.profile.profile, 'user.js') with open(userjspath) as userjsfile: prefs = userjsfile.readlines() prefs = [pref for pref in prefs if 'network.proxy' not in pref] with open(userjspath, 'w') as userjsfile: userjsfile.writelines(prefs) # for geckoview/android pageload playback we can't use a policy to turn on the # proxy; we need to set prefs instead; note that the 'host' may be different # than '127.0.0.1' so we must set the prefs accordingly if self.config['app'] == "geckoview" and test.get('playback', None) is not None: self.log.info( "setting profile prefs to turn on the geckoview browser proxy") no_proxies_on = "localhost, 127.0.0.1, %s" % self.config['host'] proxy_prefs = {} proxy_prefs["network.proxy.type"] = 1 proxy_prefs["network.proxy.http"] = self.config['host'] proxy_prefs["network.proxy.http_port"] = 8080 proxy_prefs["network.proxy.ssl"] = self.config['host'] proxy_prefs["network.proxy.ssl_port"] = 8080 proxy_prefs["network.proxy.no_proxies_on"] = no_proxies_on self.profile.set_preferences(proxy_prefs) # now some final settings, and then startup of the browser under test if self.config['app'] == "geckoview": # for android/geckoview we must copy the profile onto the device and set perms if not self.device.is_app_installed(self.config['binary']): raise Exception('%s is not installed' % self.config['binary']) self.device_profile = os.path.join(self.device_raptor_dir, "profile") if self.device.is_dir(self.device_profile): self.log.info("deleting existing device profile folder: %s" % self.device_profile) self.device.rm(self.device_profile, recursive=True) self.log.info("creating profile folder on device: %s" % self.device_profile) self.device.mkdir(self.device_profile) self.log.info("copying firefox profile onto the device") self.log.info("note: the profile folder being copied is: %s" % self.profile.profile) self.log.info( 'the adb push cmd copies that profile dir to a new temp dir before copy' ) self.device.push(self.profile.profile, self.device_profile) self.device.chmod(self.device_profile, recursive=True) # now start the geckoview app self.log.info("starting %s" % self.config['app']) extra_args = [ "-profile", self.device_profile, "--es", "env0", "LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6" ] try: # make sure the geckoview app is not running before # attempting to start. self.device.stop_application(self.config['binary']) self.device.launch_activity(self.config['binary'], "GeckoViewActivity", extra_args=extra_args, url='about:blank', e10s=True, fail_if_running=False) except Exception: self.log.error("Exception launching %s" % self.config['binary']) raise self.control_server.device = self.device self.control_server.app_name = self.config['binary'] else: # For Firefox we need to set # MOZ_DISABLE_NONLOCAL_CONNECTIONS=1 env var before # startup when testing release builds from mozilla-beta or # mozilla-release. This is because of restrictions on # release builds that require webextensions to be signed # unless MOZ_DISABLE_NONLOCAL_CONNECTIONS is set to '1'. if self.config['app'] == "firefox" and self.config[ 'is_release_build']: self.log.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=1") os.environ['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = "1" # if running debug-mode, tell Firefox to open the browser console on startup # for google chrome, open the devtools on the raptor test tab if self.debug_mode: if self.config['app'] == "firefox": self.runner.cmdargs.extend(['-jsconsole']) if self.config['app'] == "chrome": self.runner.cmdargs.extend( ['--auto-open-devtools-for-tabs']) # now start the desktop browser self.log.info("starting %s" % self.config['app']) # if running a pageload test on google chrome, add the cmd line options # to turn on the proxy and ignore security certificate errors # if using host localhost, 127.0.0.1. if self.config['app'] == "chrome" and test.get('playback', None) is not None: chrome_args = [ '--proxy-server="http=127.0.0.1:8080;' + 'https=127.0.0.1:8080;ssl=127.0.0.1:8080"', '--ignore-certificate-errors' ] if self.config['host'] not in ('localhost', '127.0.0.1'): chrome_args[0] = chrome_args[0].replace( '127.0.0.1', self.config['host']) if ' '.join(chrome_args) not in ' '.join(self.runner.cmdargs): self.runner.cmdargs.extend(chrome_args) self.runner.start() proc = self.runner.process_handler self.output_handler.proc = proc self.control_server.browser_proc = proc # pageload tests need to be able to access non-local connections via mitmproxy if self.config['app'] == "firefox" and self.config['is_release_build'] and \ test.get('playback', None) is not None: self.log.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=0") os.environ['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = "0" # if geckoProfile is enabled, initialize it if self.config['gecko_profile'] is True: self._init_gecko_profiling(test) # tell the control server the gecko_profile dir; the control server will # receive the actual gecko profiles from the web ext and will write them # to disk; then profiles are picked up by gecko_profile.symbolicate self.control_server.gecko_profile_dir = self.gecko_profiler.gecko_profile_dir # set our cs flag to indicate we are running the browser/app self.control_server._finished = False # convert to seconds and account for page cycles timeout = int(timeout / 1000) * int(test['page_cycles']) # account for the pause the raptor webext runner takes after browser startup timeout += int(self.post_startup_delay / 1000) # if geckoProfile enabled, give browser more time for profiling if self.config['gecko_profile'] is True: timeout += 5 * 60 try: elapsed_time = 0 while not self.control_server._finished: time.sleep(1) # we only want to force browser-shutdown on timeout if not in debug mode; # in debug-mode we leave the browser running (require manual shutdown) if not self.debug_mode: elapsed_time += 1 if elapsed_time > (timeout) - 5: # stop 5 seconds early self.log.info( "application timed out after {} seconds".format( timeout)) self.control_server.wait_for_quit() break finally: if self.config['app'] != "geckoview": try: self.runner.check_for_crashes() except NotImplementedError: # not implemented for Chrome pass # TODO: if on geckoview is there some cleanup here i.e. check for crashes? if self.playback is not None: self.playback.stop() # remove the raptor webext; as it must be reloaded with each subtest anyway self.log.info("removing webext %s" % raptor_webext) if self.config['app'] in ["firefox", "geckoview"]: self.profile.addons.remove_addon(webext_id) # for chrome the addon is just a list (appended to cmd line) if self.config['app'] in ["chrome", "chrome-android"]: self.profile.addons.remove(raptor_webext) # gecko profiling symbolication if self.config['gecko_profile'] is True: self.gecko_profiler.symbolicate() # clean up the temp gecko profiling folders self.log.info("cleaning up after gecko profiling") self.gecko_profiler.clean() # browser should be closed by now but this is a backup-shutdown (if not in debug-mode) if not self.debug_mode: if self.config['app'] != "geckoview": if self.runner.is_running(): self.runner.stop() # TODO the geckoview app should have been shutdown by this point by the # control server, but we can double-check here to make sure else: # in debug mode, and running locally, leave the browser running if self.config['run_local']: self.log.info( "* debug-mode enabled - please shutdown the browser manually..." ) self.runner.wait(timeout=None) def _init_gecko_profiling(self, test): self.log.info("initializing gecko profiler") upload_dir = os.getenv('MOZ_UPLOAD_DIR') if not upload_dir: self.log.critical( "Profiling ignored because MOZ_UPLOAD_DIR was not set") else: self.gecko_profiler = GeckoProfile(upload_dir, self.config, test) def process_results(self): # when running locally output results in build/raptor.json; when running # in production output to a local.json to be turned into tc job artifact if self.config.get('run_local', False): if 'MOZ_DEVELOPER_REPO_DIR' in os.environ: raptor_json_path = os.path.join( os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'mozharness', 'build', 'raptor.json') else: raptor_json_path = os.path.join(here, 'raptor.json') else: raptor_json_path = os.path.join(os.getcwd(), 'local.json') self.config['raptor_json_path'] = raptor_json_path return self.results_handler.summarize_and_output(self.config) def get_page_timeout_list(self): return self.results_handler.page_timeout_list def clean_up(self): self.control_server.stop() if self.config['app'] != "geckoview": self.runner.stop() elif self.config['app'] == 'geckoview': self.log.info('removing reverse socket connections') self.device.remove_socket_connections('reverse') else: pass self.log.info("finished")
class JUnitTestRunner(MochitestDesktop): """ A test harness to run geckoview junit tests on a remote device. """ def __init__(self, log, options): self.log = log verbose = False if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug': verbose = True self.device = ADBAndroid(adb=options.adbPath or 'adb', device=options.deviceSerial, test_root=options.remoteTestRoot, verbose=verbose) self.options = options self.log.debug("options=%s" % vars(options)) update_mozinfo() self.remote_profile = posixpath.join(self.device.test_root, 'junit-profile') if self.options.coverage and not self.options.coverage_output_path: raise Exception("--coverage-output-path is required when using --enable-coverage") self.remote_coverage_output_path = posixpath.join(self.device.test_root, 'junit-coverage.ec') self.server_init() self.cleanup() self.device.clear_logcat() self.build_profile() self.startServers( self.options, debuggerInfo=None, ignoreSSLTunnelExts=True) self.log.debug("Servers started") def server_init(self): """ Additional initialization required to satisfy MochitestDesktop.startServers """ self._locations = None self.server = None self.wsserver = None self.websocketProcessBridge = None self.SERVER_STARTUP_TIMEOUT = 180 if mozinfo.info.get('debug') else 90 if self.options.remoteWebServer is None: if os.name != "nt": self.options.remoteWebServer = moznetwork.get_ip() else: raise Exception("--remote-webserver must be specified") self.options.webServer = self.options.remoteWebServer self.options.webSocketPort = '9988' self.options.httpdPath = None self.options.keep_open = False self.options.pidFile = "" self.options.subsuite = None self.options.xrePath = None if build_obj and 'MOZ_HOST_BIN' in os.environ: self.options.xrePath = os.environ['MOZ_HOST_BIN'] if not self.options.utilityPath: self.options.utilityPath = self.options.xrePath if not self.options.xrePath: self.options.xrePath = self.options.utilityPath if build_obj: self.options.certPath = os.path.join(build_obj.topsrcdir, 'build', 'pgo', 'certs') def build_profile(self): """ Create a local profile with test prefs and proxy definitions and push it to the remote device. """ self.profile = Profile(locations=self.locations, proxy=self.proxy(self.options)) self.options.profilePath = self.profile.profile # Set preferences self.merge_base_profiles(self.options) if self.fillCertificateDB(self.options): self.log.error("Certificate integration failed") self.device.mkdir(self.remote_profile, parents=True) self.device.push(self.profile.profile, self.remote_profile) self.log.debug("profile %s -> %s" % (str(self.profile.profile), str(self.remote_profile))) def cleanup(self): try: self.stopServers() self.log.debug("Servers stopped") self.device.stop_application(self.options.app) self.device.rm(self.remote_profile, force=True, recursive=True) if hasattr(self, 'profile'): del self.profile except Exception: traceback.print_exc() self.log.info("Caught and ignored an exception during cleanup") def build_command_line(self, test_filters): """ Construct and return the 'am instrument' command line. """ cmd = "am instrument -w -r" # profile location cmd = cmd + " -e args '-profile %s'" % self.remote_profile # multi-process e10s = 'true' if self.options.e10s else 'false' cmd = cmd + " -e use_multiprocess %s" % e10s # chunks (shards) shards = self.options.totalChunks shard = self.options.thisChunk if shards is not None and shard is not None: shard -= 1 # shard index is 0 based cmd = cmd + " -e numShards %d -e shardIndex %d" % (shards, shard) # test filters: limit run to specific test(s) for f in test_filters: # filter can be class-name or 'class-name#method-name' (single test) cmd = cmd + " -e class %s" % f # enable code coverage reports if self.options.coverage: cmd = cmd + " -e coverage true" cmd = cmd + " -e coverageFile %s" % self.remote_coverage_output_path # environment env = {} env["MOZ_CRASHREPORTER"] = "1" env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" env["XPCOM_DEBUG_BREAK"] = "stack" env["DISABLE_UNSAFE_CPOW_WARNINGS"] = "1" env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" env["MOZ_IN_AUTOMATION"] = "1" env["R_LOG_VERBOSE"] = "1" env["R_LOG_LEVEL"] = "6" env["R_LOG_DESTINATION"] = "stderr" for (env_count, (env_key, env_val)) in enumerate(env.iteritems()): cmd = cmd + " -e env%d %s=%s" % (env_count, env_key, env_val) # runner cmd = cmd + " %s/%s" % (self.options.app, self.options.runner) return cmd @property def locations(self): if self._locations is not None: return self._locations locations_file = os.path.join(here, 'server-locations.txt') self._locations = ServerLocations(locations_file) return self._locations def run_tests(self, test_filters=None): """ Run the tests. """ if not self.device.is_app_installed(self.options.app): raise Exception("%s is not installed" % self.options.app) if self.device.process_exist(self.options.app): raise Exception("%s already running before starting tests" % self.options.app) self.test_started = False self.pass_count = 0 self.fail_count = 0 self.todo_count = 0 self.class_name = "" self.test_name = "" self.current_full_name = "" def callback(line): # Output callback: Parse the raw junit log messages, translating into # treeherder-friendly test start/pass/fail messages. self.log.process_output(self.options.app, str(line)) # Expect per-test info like: "INSTRUMENTATION_STATUS: class=something" match = re.match(r'INSTRUMENTATION_STATUS:\s*class=(.*)', line) if match: self.class_name = match.group(1) # Expect per-test info like: "INSTRUMENTATION_STATUS: test=something" match = re.match(r'INSTRUMENTATION_STATUS:\s*test=(.*)', line) if match: self.test_name = match.group(1) # Expect per-test info like: "INSTRUMENTATION_STATUS_CODE: 0|1|..." match = re.match(r'INSTRUMENTATION_STATUS_CODE:\s*([+-]?\d+)', line) if match: status = match.group(1) full_name = "%s.%s" % (self.class_name, self.test_name) if full_name == self.current_full_name: if status == '0': message = '' status = 'PASS' expected = 'PASS' self.pass_count += 1 elif status == '-3': # ignored (skipped) message = '' status = 'SKIP' expected = 'SKIP' self.todo_count += 1 elif status == '-4': # known fail message = '' status = 'FAIL' expected = 'FAIL' self.todo_count += 1 else: message = 'status %s' % status status = 'FAIL' expected = 'PASS' self.fail_count += 1 self.log.test_end(full_name, status, expected, message) self.test_started = False else: if self.test_started: # next test started without reporting previous status self.fail_count += 1 status = 'FAIL' expected = 'PASS' self.log.test_end(self.current_full_name, status, expected, "missing test completion status") self.log.test_start(full_name) self.test_started = True self.current_full_name = full_name # Ideally all test names should be reported to suite_start, but these test # names are not known in advance. self.log.suite_start(["geckoview-junit"]) try: cmd = self.build_command_line(test_filters) self.log.info("launching %s" % cmd) p = self.device.shell(cmd, timeout=self.options.max_time, stdout_callback=callback) if p.timedout: self.log.error("TEST-UNEXPECTED-TIMEOUT | runjunit.py | " "Timed out after %d seconds" % self.options.max_time) self.log.info("Passed: %d" % self.pass_count) self.log.info("Failed: %d" % self.fail_count) self.log.info("Todo: %d" % self.todo_count) finally: self.log.suite_end() if self.check_for_crashes(): self.fail_count = 1 if self.options.coverage: self.device.pull(self.remote_coverage_output_path, self.options.coverage_output_path) return 1 if self.fail_count else 0 def check_for_crashes(self): logcat = self.device.get_logcat() if logcat: if mozcrash.check_for_java_exception(logcat, self.current_full_name): return True symbols_path = self.options.symbolsPath try: dump_dir = tempfile.mkdtemp() remote_dir = posixpath.join(self.remote_profile, 'minidumps') if not self.device.is_dir(remote_dir): # If crash reporting is enabled (MOZ_CRASHREPORTER=1), the # minidumps directory is automatically created when the app # (first) starts, so its lack of presence is a hint that # something went wrong. print "Automation Error: No crash directory (%s) found on remote device" % \ remote_dir return True self.device.pull(remote_dir, dump_dir) crashed = mozcrash.log_crashes(self.log, dump_dir, symbols_path, test=self.current_full_name) finally: try: shutil.rmtree(dump_dir) except Exception: self.log.warning("unable to remove directory: %s" % dump_dir) return crashed
class GeckoviewTestRunner: """ A quick-and-dirty test harness to verify the geckoview_example app starts without crashing. """ def __init__(self, log, options): self.log = log self.device = ADBAndroid(adb=options.adbPath or 'adb', device=options.deviceSerial, test_root=options.remoteTestRoot) self.options = options self.appname = self.options.app.split('/')[-1] self.logcat = None self.build_profile() self.log.debug("options=%s" % vars(options)) def build_profile(self): test_root = self.device.test_root self.remote_profile = posixpath.join(test_root, 'gv-profile') self.device.mkdir(self.remote_profile, parents=True) profile = Profile() self.device.push(profile.profile, self.remote_profile) self.log.debug("profile %s -> %s" % (str(profile.profile), str(self.remote_profile))) def installed(self): """ geckoview_example installed """ if not self.device.is_app_installed(self.appname): return (False, "%s not installed" % self.appname) return (True, "%s installed" % self.appname) def start(self): """ geckoview_example starts """ try: self.device.stop_application(self.appname) self.device.clear_logcat() args = ["-profile", self.remote_profile] env = {} env["MOZ_CRASHREPORTER"] = "1" env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" env["XPCOM_DEBUG_BREAK"] = "stack" env["DISABLE_UNSAFE_CPOW_WARNINGS"] = "1" env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" env["MOZ_IN_AUTOMATION"] = "1" env["R_LOG_VERBOSE"] = "1" env["R_LOG_LEVEL"] = "6" env["R_LOG_DESTINATION"] = "stderr" self.device.launch_activity("org.mozilla.geckoview_example", "GeckoViewActivity", extra_args=args, moz_env=env) except Exception: return (False, "Exception during %s startup" % self.appname) return (True, "%s started" % self.appname) def started(self): """ startup logcat messages """ expected = [ "zerdatime", "Displayed %s/.GeckoViewActivity" % self.appname ] # wait up to 60 seconds for startup for wait_time in xrange(60): time.sleep(1) self.logcat = self.device.get_logcat() for line in self.logcat: for e in expected: if e in line: self.log.debug(line.strip()) expected.remove(e) if len(expected) == 0: return (True, "All expected logcat messages found") for e in expected: self.log.error("missing from logcat: '%s'" % e) return (False, "'%s' not found in logcat" % expected[0]) def run_tests(self): """ Run simple tests to verify that the geckoview_example app starts. """ all_tests = [self.installed, self.start, self.started] self.log.suite_start(all_tests) pass_count = 0 fail_count = 0 for test in all_tests: self.test_name = test.__doc__.strip() self.log.test_start(self.test_name) expected = 'PASS' (passed, message) = test() if passed: pass_count += 1 else: fail_count += 1 status = 'PASS' if passed else 'FAIL' self.log.test_end(self.test_name, status, expected, message) crashed = self.check_for_crashes() if crashed: fail_count = 1 else: self.log.info("Passed: %d" % pass_count) self.log.info("Failed: %d" % fail_count) self.log.suite_end() return 1 if fail_count else 0 def check_for_crashes(self): if self.logcat: if mozcrash.check_for_java_exception(self.logcat, self.test_name): return True symbols_path = self.options.symbols_path try: dump_dir = tempfile.mkdtemp() remote_dir = posixpath.join(self.remote_profile, 'minidumps') crash_dir_found = False # wait up to 60 seconds for gecko startup to progress through # crashreporter initialization, in case all tests finished quickly for wait_time in xrange(60): time.sleep(1) if self.device.is_dir(remote_dir): crash_dir_found = True break if not crash_dir_found: # If crash reporting is enabled (MOZ_CRASHREPORTER=1), the # minidumps directory is automatically created when the app # (first) starts, so its lack of presence is a hint that # something went wrong. print "Automation Error: No crash directory (%s) found on remote device" % \ remote_dir # Whilst no crash was found, the run should still display as a failure return True self.device.pull(remote_dir, dump_dir) crashed = mozcrash.log_crashes(self.log, dump_dir, symbols_path, test=self.test_name) finally: try: shutil.rmtree(dump_dir) except Exception: self.log.warning("unable to remove directory: %s" % dump_dir) return crashed def cleanup(self): """ Cleanup at end of job run. """ self.log.debug("Cleaning up...") self.device.stop_application(self.appname) self.device.rm(self.remote_profile, force=True, recursive=True) self.log.debug("Cleanup complete.")