def run_tests_remote(tests, num_tests, prefix, options, slog): # Setup device with everything needed to run our tests. from mozdevice import ADBAndroid device = ADBAndroid(device=options.device_serial, test_root=options.remote_test_root) # Update the test root to point to our test directory. jit_tests_dir = posixpath.join(options.remote_test_root, 'jit-tests') options.remote_test_root = posixpath.join(jit_tests_dir, 'jit-tests') # Push js shell and libraries. device.rm(jit_tests_dir, force=True, recursive=True) device.mkdir(options.remote_test_root, parents=True) push_libs(options, device) push_progs(options, device, [prefix[0]]) device.chmod(options.remote_test_root, recursive=True, root=True) JitTest.CacheDir = posixpath.join(options.remote_test_root, '.js-cache') device.mkdir(JitTest.CacheDir) device.push(JS_TESTS_DIR, posixpath.join(jit_tests_dir, 'tests'), timeout=600) device.push(os.path.dirname(TEST_DIR), options.remote_test_root, timeout=600) prefix[0] = os.path.join(options.remote_test_root, 'js') # Run all tests. pb = create_progressbar(num_tests, options) gen = get_remote_results(tests, device, prefix, options) ok = process_test_results(gen, num_tests, pb, options, slog) return ok
class Raptor(object): """Container class for Raptor""" def __init__(self, app, binary, run_local=False, obj_path=None): self.config = {} self.config['app'] = app self.config['binary'] = binary self.config['platform'] = mozinfo.os self.config['processor'] = mozinfo.processor self.config['run_local'] = run_local self.config['obj_path'] = obj_path self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv') self.log = get_default_logger(component='raptor-main') self.control_server = None self.playback = None self.benchmark = None # Create the profile; for geckoview we want a firefox profile type if self.config['app'] == 'geckoview': self.profile = create_profile('firefox') else: self.profile = create_profile(self.config['app']) # Merge in base profiles with open(os.path.join(self.profile_data_dir, 'profiles.json'), 'r') as fh: base_profiles = json.load(fh)['raptor'] for name in base_profiles: path = os.path.join(self.profile_data_dir, name) self.log.info("Merging profile: {}".format(path)) self.profile.merge(path) # create results holder self.results_handler = RaptorResultsHandler() # when testing desktop browsers we use mozrunner to start the browser; when # testing on android (i.e. geckoview) we use mozdevice to control the device app if self.config['app'] == "geckoview": # create the android device handler; it gets initiated and sets up adb etc self.log.info("creating android device handler using mozdevice") self.device = ADBAndroid(verbose=True) self.device.clear_logcat() else: # create the desktop browser runner self.log.info("creating browser runner using mozrunner") self.output_handler = OutputHandler() process_args = { 'processOutputLine': [self.output_handler], } runner_cls = runners[app] self.runner = runner_cls(binary, profile=self.profile, process_args=process_args) self.log.info("raptor config: %s" % str(self.config)) @property def profile_data_dir(self): if 'MOZ_DEVELOPER_REPO_DIR' in os.environ: return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'profiles') if build: return os.path.join(build.topsrcdir, 'testing', 'profiles') return os.path.join(here, 'profile_data') def start_control_server(self): self.control_server = RaptorControlServer(self.results_handler) self.control_server.start() # for android we must make the control server available to the device if self.config['app'] == "geckoview": self.log.info( "making the raptor control server port available to device") _tcp_port = "tcp:%s" % self.control_server.port self.device.create_socket_connection('reverse', _tcp_port, _tcp_port) def get_playback_config(self, test): self.config['playback_tool'] = test.get('playback') self.log.info("test uses playback tool: %s " % self.config['playback_tool']) self.config['playback_binary_manifest'] = test.get( 'playback_binary_manifest', None) _key = 'playback_binary_zip_%s' % self.config['platform'] self.config['playback_binary_zip'] = test.get(_key, None) self.config['playback_pageset_manifest'] = test.get( 'playback_pageset_manifest', None) _key = 'playback_pageset_zip_%s' % self.config['platform'] self.config['playback_pageset_zip'] = test.get(_key, None) self.config['playback_recordings'] = test.get('playback_recordings', None) self.config['python3_win_manifest'] = test.get('python3_win_manifest', None) def run_test(self, test, timeout=None): self.log.info("starting raptor test: %s" % test['name']) self.log.info("test settings: %s" % str(test)) self.log.info("raptor config: %s" % str(self.config)) # benchmark-type tests require the benchmark test to be served out if test.get('type') == "benchmark": self.benchmark = Benchmark(self.config, test) benchmark_port = int(self.benchmark.port) else: benchmark_port = 0 gen_test_config(self.config['app'], test['name'], self.control_server.port, benchmark_port) # for android we must make the benchmarks server available to the device if self.config['app'] == "geckoview": self.log.info( "making the raptor benchmarks server port available to device") _tcp_port = "tcp:%s" % benchmark_port self.device.create_socket_connection('reverse', _tcp_port, _tcp_port) # must intall raptor addon each time because we dynamically update some content raptor_webext = os.path.join(webext_dir, 'raptor') self.log.info("installing webext %s" % raptor_webext) self.profile.addons.install(raptor_webext) # add test specific preferences if test.get("preferences", None) is not None: if self.config['app'] == "firefox": self.profile.set_preferences(json.loads(test['preferences'])) else: self.log.info("preferences were configured for the test, \ but we do not install them on non Firefox browsers." ) # on firefox we can get an addon id; chrome addon actually is just cmd line arg if self.config['app'] in ["firefox", "geckoview"]: webext_id = self.profile.addons.addon_details(raptor_webext)['id'] # some tests require tools to playback the test pages if test.get('playback', None) is not None: self.get_playback_config(test) # startup the playback tool self.playback = get_playback(self.config) # for geckoview we must copy the profile onto the device and set perms if self.config['app'] == "geckoview": if not self.device.is_app_installed(self.config['binary']): raise Exception('%s is not installed' % self.config['binary']) self.log.info("copying firefox profile onto the android device") self.device_profile = "/sdcard/raptor-profile" if self.device.is_dir(self.device_profile): self.device.rm(self.device_profile, recursive=True) self.device.mkdir(self.device_profile) self.device.push(self.profile.profile, self.device_profile) self.log.info("setting permisions to profile dir on the device") self.device.chmod(self.device_profile, recursive=True) # now start the geckoview app self.log.info("starting %s" % self.config['app']) extra_args = [ "-profile", self.device_profile, "--es", "env0", "LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6" ] try: # make sure the geckoview app is not running before # attempting to start. self.device.stop_application(self.config['binary']) self.device.launch_activity(self.config['binary'], "GeckoViewActivity", extra_args=extra_args, url='about:blank', fail_if_running=False) except Exception: self.log.error("Exception launching %s" % self.config['binary']) raise self.control_server.device = self.device self.control_server.app_name = self.config['binary'] else: # now start the desktop browser self.log.info("starting %s" % self.config['app']) self.runner.start() proc = self.runner.process_handler self.output_handler.proc = proc self.control_server.browser_proc = proc # set our cs flag to indicate we are running the browser/app self.control_server._finished = False # convert to seconds and account for page cycles timeout = int(timeout / 1000) * int(test['page_cycles']) try: elapsed_time = 0 while not self.control_server._finished: time.sleep(1) elapsed_time += 1 if elapsed_time > (timeout) - 5: # stop 5 seconds early self.log.info( "application timed out after {} seconds".format( timeout)) self.control_server.wait_for_quit() break finally: if self.config['app'] != "geckoview": try: self.runner.check_for_crashes() except NotImplementedError: # not implemented for Chrome pass # TODO: if on geckoview is there some cleanup here i.e. check for crashes? if self.playback is not None: self.playback.stop() # remove the raptor webext; as it must be reloaded with each subtest anyway # applies to firefox only; chrome the addon is actually just cmd line arg if self.config['app'] in ["firefox", "geckoview"]: self.log.info("removing webext %s" % raptor_webext) self.profile.addons.remove_addon(webext_id) if self.config['app'] != "geckoview": if self.runner.is_running(): self.runner.stop() # TODO the geckoview app should have been shutdown by this point by the # control server, but we can double-check here to make sure def process_results(self): # when running locally output results in build/raptor.json; when running # in production output to a local.json to be turned into tc job artifact if self.config.get('run_local', False): if 'MOZ_DEVELOPER_REPO_DIR' in os.environ: raptor_json_path = os.path.join( os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'mozharness', 'build', 'raptor.json') else: raptor_json_path = os.path.join(here, 'raptor.json') else: raptor_json_path = os.path.join(os.getcwd(), 'local.json') self.config['raptor_json_path'] = raptor_json_path return self.results_handler.summarize_and_output(self.config) def clean_up(self): self.control_server.stop() if self.config['app'] != "geckoview": self.runner.stop() elif self.config['app'] == 'geckoview': self.log.info('removing reverse socket connections') self.device.remove_socket_connections('reverse') else: pass self.log.info("finished")
class RemoteReftest(RefTest): use_marionette = False resolver_cls = RemoteReftestResolver def __init__(self, options, scriptDir): RefTest.__init__(self, options.suite) self.run_by_manifest = False self.scriptDir = scriptDir self.localLogName = options.localLogName verbose = False if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug': verbose = True print "set verbose!" self.device = ADBAndroid(adb=options.adb_path, device=options.deviceSerial, test_root=options.remoteTestRoot, verbose=verbose) if options.remoteTestRoot is None: options.remoteTestRoot = posixpath.join(self.device.test_root, "reftest") options.remoteProfile = posixpath.join(options.remoteTestRoot, "profile") options.remoteLogFile = posixpath.join(options.remoteTestRoot, "reftest.log") options.logFile = options.remoteLogFile self.remoteProfile = options.remoteProfile self.remoteTestRoot = options.remoteTestRoot if not options.ignoreWindowSize: parts = self.device.get_info('screen')['screen'][0].split() width = int(parts[0].split(':')[1]) height = int(parts[1].split(':')[1]) if (width < 1366 or height < 1050): self.error("ERROR: Invalid screen resolution %sx%s, " "please adjust to 1366x1050 or higher" % (width, height)) self._populate_logger(options) self.outputHandler = OutputHandler(self.log, options.utilityPath, options.symbolsPath) # RemoteAutomation.py's 'messageLogger' is also used by mochitest. Mimic a mochitest # MessageLogger object to re-use this code path. self.outputHandler.write = self.outputHandler.__call__ self.automation = RemoteAutomation(self.device, options.app, self.remoteProfile, options.remoteLogFile, processArgs=None) self.automation._processArgs['messageLogger'] = self.outputHandler self.environment = self.automation.environment if self.automation.IS_DEBUG_BUILD: self.SERVER_STARTUP_TIMEOUT = 180 else: self.SERVER_STARTUP_TIMEOUT = 90 self.remoteCache = os.path.join(options.remoteTestRoot, "cache/") # Check that Firefox is installed expected = options.app.split('/')[-1] if not self.device.is_app_installed(expected): raise Exception("%s is not installed on this device" % expected) self.automation.deleteANRs() self.automation.deleteTombstones() self.device.clear_logcat() self.device.rm(self.remoteCache, force=True, recursive=True) procName = options.app.split('/')[-1] self.device.pkill(procName) if self.device.process_exist(procName): self.log.error("unable to kill %s before starting tests!" % procName) def findPath(self, paths, filename=None): for path in paths: p = path if filename: p = os.path.join(p, filename) if os.path.exists(self.getFullPath(p)): return path return None def startWebServer(self, options): """ Create the webserver on the host and start it up """ remoteXrePath = options.xrePath remoteUtilityPath = options.utilityPath localAutomation = Automation() localAutomation.IS_WIN32 = False localAutomation.IS_LINUX = False localAutomation.IS_MAC = False localAutomation.UNIXISH = False hostos = sys.platform if (hostos == 'mac' or hostos == 'darwin'): localAutomation.IS_MAC = True elif (hostos == 'linux' or hostos == 'linux2'): localAutomation.IS_LINUX = True localAutomation.UNIXISH = True elif (hostos == 'win32' or hostos == 'win64'): localAutomation.BIN_SUFFIX = ".exe" localAutomation.IS_WIN32 = True paths = [options.xrePath, localAutomation.DIST_BIN] options.xrePath = self.findPath(paths) if options.xrePath is None: print( "ERROR: unable to find xulrunner path for %s, " "please specify with --xre-path" % (os.name)) return 1 paths.append("bin") paths.append(os.path.join("..", "bin")) xpcshell = "xpcshell" if (os.name == "nt"): xpcshell += ".exe" if (options.utilityPath): paths.insert(0, options.utilityPath) options.utilityPath = self.findPath(paths, xpcshell) if options.utilityPath is None: print( "ERROR: unable to find utility path for %s, " "please specify with --utility-path" % (os.name)) return 1 options.serverProfilePath = tempfile.mkdtemp() self.server = ReftestServer(localAutomation, options, self.scriptDir) retVal = self.server.start() if retVal: return retVal retVal = self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT) if retVal: return retVal options.xrePath = remoteXrePath options.utilityPath = remoteUtilityPath return 0 def stopWebServer(self, options): self.server.stop() def killNamedProc(self, pname, orphans=True): """ Kill processes matching the given command name """ self.log.info("Checking for %s processes..." % pname) for proc in psutil.process_iter(): try: if proc.name() == pname: procd = proc.as_dict( attrs=['pid', 'ppid', 'name', 'username']) if proc.ppid() == 1 or not orphans: self.log.info("killing %s" % procd) try: os.kill(proc.pid, getattr(signal, "SIGKILL", signal.SIGTERM)) except Exception as e: self.log.info("Failed to kill process %d: %s" % (proc.pid, str(e))) else: self.log.info("NOT killing %s (not an orphan?)" % procd) except Exception: # may not be able to access process info for all processes continue def createReftestProfile(self, options, **kwargs): profile = RefTest.createReftestProfile(self, options, server=options.remoteWebServer, port=options.httpPort, **kwargs) profileDir = profile.profile prefs = {} prefs["app.update.url.android"] = "" prefs["browser.firstrun.show.localepicker"] = False prefs["reftest.remote"] = True prefs[ "datareporting.policy.dataSubmissionPolicyBypassAcceptance"] = True # move necko cache to a location that can be cleaned up prefs["browser.cache.disk.parent_directory"] = self.remoteCache prefs["layout.css.devPixelsPerPx"] = "1.0" # Because Fennec is a little wacky (see bug 1156817) we need to load the # reftest pages at 1.0 zoom, rather than zooming to fit the CSS viewport. prefs["apz.allow_zooming"] = False # Set the extra prefs. profile.set_preferences(prefs) try: self.device.push(profileDir, options.remoteProfile) self.device.chmod(options.remoteProfile, recursive=True) except Exception: print "Automation Error: Failed to copy profiledir to device" raise return profile def copyExtraFilesToProfile(self, options, profile): profileDir = profile.profile RefTest.copyExtraFilesToProfile(self, options, profile) if len(os.listdir(profileDir)) > 0: try: self.device.push(profileDir, options.remoteProfile) self.device.chmod(options.remoteProfile, recursive=True) except Exception: print "Automation Error: Failed to copy extra files to device" raise def printDeviceInfo(self, printLogcat=False): try: if printLogcat: logcat = self.device.get_logcat( filter_out_regexps=fennecLogcatFilters) print ''.join(logcat) print "Device info:" devinfo = self.device.get_info() for category in devinfo: if type(devinfo[category]) is list: print " %s:" % category for item in devinfo[category]: print " %s" % item else: print " %s: %s" % (category, devinfo[category]) print "Test root: %s" % self.device.test_root except Exception as e: print "WARNING: Error getting device information: %s" % str(e) def environment(self, **kwargs): return self.automation.environment(**kwargs) def buildBrowserEnv(self, options, profileDir): browserEnv = RefTest.buildBrowserEnv(self, options, profileDir) # remove desktop environment not used on device if "XPCOM_MEM_BLOAT_LOG" in browserEnv: del browserEnv["XPCOM_MEM_BLOAT_LOG"] return browserEnv def runApp(self, options, cmdargs=None, timeout=None, debuggerInfo=None, symbolsPath=None, valgrindPath=None, valgrindArgs=None, valgrindSuppFiles=None, **profileArgs): if cmdargs is None: cmdargs = [] if self.use_marionette: cmdargs.append('-marionette') binary = options.app profile = self.createReftestProfile(options, **profileArgs) # browser environment env = self.buildBrowserEnv(options, profile.profile) self.log.info("Running with e10s: {}".format(options.e10s)) status, self.lastTestSeen = self.automation.runApp( None, env, binary, profile.profile, cmdargs, utilityPath=options.utilityPath, xrePath=options.xrePath, debuggerInfo=debuggerInfo, symbolsPath=symbolsPath, timeout=timeout) self.cleanup(profile.profile) return status def cleanup(self, profileDir): self.device.rm(self.remoteTestRoot, force=True, recursive=True) self.device.rm(self.remoteProfile, force=True, recursive=True) self.device.rm(self.remoteCache, force=True, recursive=True) RefTest.cleanup(self, profileDir)
class MochiRemote(MochitestDesktop): localProfile = None logMessages = [] def __init__(self, options): MochitestDesktop.__init__(self, options.flavor, vars(options)) verbose = False if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug': verbose = True if hasattr(options, 'log'): delattr(options, 'log') self.certdbNew = True self.chromePushed = False self.mozLogName = "moz.log" self.device = ADBAndroid(adb=options.adbPath, device=options.deviceSerial, test_root=options.remoteTestRoot, verbose=verbose) if options.remoteTestRoot is None: options.remoteTestRoot = self.device.test_root options.dumpOutputDirectory = options.remoteTestRoot self.remoteLogFile = posixpath.join(options.remoteTestRoot, "logs", "mochitest.log") logParent = posixpath.dirname(self.remoteLogFile) self.device.rm(logParent, force=True, recursive=True) self.device.mkdir(logParent) self.remoteProfile = posixpath.join(options.remoteTestRoot, "profile/") self.device.rm(self.remoteProfile, force=True, recursive=True) self.counts = dict() self.message_logger = MessageLogger(logger=None) self.message_logger.logger = self.log process_args = { 'messageLogger': self.message_logger, 'counts': self.counts } self.automation = RemoteAutomation(self.device, options.remoteappname, self.remoteProfile, self.remoteLogFile, processArgs=process_args) self.environment = self.automation.environment # Check that Firefox is installed expected = options.app.split('/')[-1] if not self.device.is_app_installed(expected): raise Exception("%s is not installed on this device" % expected) self.automation.deleteANRs() self.automation.deleteTombstones() self.device.clear_logcat() self.remoteModulesDir = posixpath.join(options.remoteTestRoot, "modules/") self.remoteCache = posixpath.join(options.remoteTestRoot, "cache/") self.device.rm(self.remoteCache, force=True, recursive=True) # move necko cache to a location that can be cleaned up options.extraPrefs += [ "browser.cache.disk.parent_directory=%s" % self.remoteCache ] self.remoteMozLog = posixpath.join(options.remoteTestRoot, "mozlog") self.device.rm(self.remoteMozLog, force=True, recursive=True) self.device.mkdir(self.remoteMozLog) self.remoteChromeTestDir = posixpath.join(options.remoteTestRoot, "chrome") self.device.rm(self.remoteChromeTestDir, force=True, recursive=True) self.device.mkdir(self.remoteChromeTestDir) procName = options.app.split('/')[-1] self.device.stop_application(procName) if self.device.process_exist(procName): self.log.warning("unable to kill %s before running tests!" % procName) # Add Android version (SDK level) to mozinfo so that manifest entries # can be conditional on android_version. self.log.info( "Android sdk version '%s'; will use this to filter manifests" % str(self.device.version)) mozinfo.info['android_version'] = str(self.device.version) def cleanup(self, options, final=False): if final: self.device.rm(self.remoteChromeTestDir, force=True, recursive=True) self.chromePushed = False uploadDir = os.environ.get('MOZ_UPLOAD_DIR', None) if uploadDir and self.device.is_dir(self.remoteMozLog): self.device.pull(self.remoteMozLog, uploadDir) self.device.rm(self.remoteLogFile, force=True) self.device.rm(self.remoteProfile, force=True, recursive=True) self.device.rm(self.remoteCache, force=True, recursive=True) MochitestDesktop.cleanup(self, options, final) self.localProfile = None def findPath(self, paths, filename=None): for path in paths: p = path if filename: p = os.path.join(p, filename) if os.path.exists(self.getFullPath(p)): return path return None def makeLocalAutomation(self): localAutomation = Automation() localAutomation.IS_WIN32 = False localAutomation.IS_LINUX = False localAutomation.IS_MAC = False localAutomation.UNIXISH = False hostos = sys.platform if (hostos == 'mac' or hostos == 'darwin'): localAutomation.IS_MAC = True elif (hostos == 'linux' or hostos == 'linux2'): localAutomation.IS_LINUX = True localAutomation.UNIXISH = True elif (hostos == 'win32' or hostos == 'win64'): localAutomation.BIN_SUFFIX = ".exe" localAutomation.IS_WIN32 = True return localAutomation # This seems kludgy, but this class uses paths from the remote host in the # options, except when calling up to the base class, which doesn't # understand the distinction. This switches out the remote values for local # ones that the base class understands. This is necessary for the web # server, SSL tunnel and profile building functions. def switchToLocalPaths(self, options): """ Set local paths in the options, return a function that will restore remote values """ remoteXrePath = options.xrePath remoteProfilePath = options.profilePath remoteUtilityPath = options.utilityPath localAutomation = self.makeLocalAutomation() paths = [ options.xrePath, localAutomation.DIST_BIN, ] options.xrePath = self.findPath(paths) if options.xrePath is None: self.log.error( "unable to find xulrunner path for %s, please specify with --xre-path" % os.name) sys.exit(1) xpcshell = "xpcshell" if (os.name == "nt"): xpcshell += ".exe" if options.utilityPath: paths = [options.utilityPath, options.xrePath] else: paths = [options.xrePath] options.utilityPath = self.findPath(paths, xpcshell) if options.utilityPath is None: self.log.error( "unable to find utility path for %s, please specify with --utility-path" % os.name) sys.exit(1) xpcshell_path = os.path.join(options.utilityPath, xpcshell) if localAutomation.elf_arm(xpcshell_path): self.log.error('xpcshell at %s is an ARM binary; please use ' 'the --utility-path argument to specify the path ' 'to a desktop version.' % xpcshell_path) sys.exit(1) if self.localProfile: options.profilePath = self.localProfile else: options.profilePath = None def fixup(): options.xrePath = remoteXrePath options.utilityPath = remoteUtilityPath options.profilePath = remoteProfilePath return fixup def startServers(self, options, debuggerInfo): """ Create the servers on the host and start them up """ restoreRemotePaths = self.switchToLocalPaths(options) # ignoreSSLTunnelExts is a workaround for bug 1109310 MochitestDesktop.startServers(self, options, debuggerInfo, ignoreSSLTunnelExts=True) restoreRemotePaths() def buildProfile(self, options): restoreRemotePaths = self.switchToLocalPaths(options) if options.testingModulesDir: try: self.device.push(options.testingModulesDir, self.remoteModulesDir) self.device.chmod(self.remoteModulesDir, recursive=True) except Exception: self.log.error( "Automation Error: Unable to copy test modules to device.") raise savedTestingModulesDir = options.testingModulesDir options.testingModulesDir = self.remoteModulesDir else: savedTestingModulesDir = None manifest = MochitestDesktop.buildProfile(self, options) if savedTestingModulesDir: options.testingModulesDir = savedTestingModulesDir self.localProfile = options.profilePath restoreRemotePaths() options.profilePath = self.remoteProfile return manifest def buildURLOptions(self, options, env): saveLogFile = options.logFile options.logFile = self.remoteLogFile options.profilePath = self.localProfile env["MOZ_HIDE_RESULTS_TABLE"] = "1" retVal = MochitestDesktop.buildURLOptions(self, options, env) # we really need testConfig.js (for browser chrome) try: self.device.push(options.profilePath, self.remoteProfile) self.device.chmod(self.remoteProfile, recursive=True) except Exception: self.log.error( "Automation Error: Unable to copy profile to device.") raise options.profilePath = self.remoteProfile options.logFile = saveLogFile return retVal def getChromeTestDir(self, options): local = super(MochiRemote, self).getChromeTestDir(options) remote = self.remoteChromeTestDir if options.flavor == 'chrome' and not self.chromePushed: self.log.info("pushing %s to %s on device..." % (local, remote)) local = os.path.join(local, "chrome") self.device.push(local, remote) self.chromePushed = True return remote def getLogFilePath(self, logFile): return logFile def printDeviceInfo(self, printLogcat=False): try: if printLogcat: logcat = self.device.get_logcat( filter_out_regexps=fennecLogcatFilters) self.log.info('\n' + ''.join(logcat).decode('utf-8', 'replace')) self.log.info("Device info:") devinfo = self.device.get_info() for category in devinfo: if type(devinfo[category]) is list: self.log.info(" %s:" % category) for item in devinfo[category]: self.log.info(" %s" % item) else: self.log.info(" %s: %s" % (category, devinfo[category])) self.log.info("Test root: %s" % self.device.test_root) except Exception as e: self.log.warning("Error getting device information: %s" % str(e)) def getGMPPluginPath(self, options): # TODO: bug 1149374 return None def buildBrowserEnv(self, options, debugger=False): browserEnv = MochitestDesktop.buildBrowserEnv(self, options, debugger=debugger) # remove desktop environment not used on device if "XPCOM_MEM_BLOAT_LOG" in browserEnv: del browserEnv["XPCOM_MEM_BLOAT_LOG"] # override mozLogs to avoid processing in MochitestDesktop base class self.mozLogs = None browserEnv["MOZ_LOG_FILE"] = os.path.join(self.remoteMozLog, self.mozLogName) if options.dmd: browserEnv['DMD'] = '1' return browserEnv def runApp(self, *args, **kwargs): """front-end automation.py's `runApp` functionality until FennecRunner is written""" # automation.py/remoteautomation `runApp` takes the profile path, # whereas runtest.py's `runApp` takes a mozprofile object. if 'profileDir' not in kwargs and 'profile' in kwargs: kwargs['profileDir'] = kwargs.pop('profile').profile # remove args not supported by automation.py kwargs.pop('marionette_args', None) ret, _ = self.automation.runApp(*args, **kwargs) self.countpass += self.counts['pass'] self.countfail += self.counts['fail'] self.counttodo += self.counts['todo'] return ret, None
class RemoteCPPUnitTests(cppunittests.CPPUnitTests): def __init__(self, options, progs): cppunittests.CPPUnitTests.__init__(self) self.options = options self.device = ADBAndroid(adb=options.adb_path or 'adb', device=options.device_serial, test_root=options.remote_test_root) self.remote_test_root = posixpath.join(self.device.test_root, "cppunittests") self.remote_bin_dir = posixpath.join(self.remote_test_root, "b") self.remote_tmp_dir = posixpath.join(self.remote_test_root, "tmp") self.remote_home_dir = posixpath.join(self.remote_test_root, "h") if options.setup: self.setup_bin(progs) def setup_bin(self, progs): self.device.rm(self.remote_test_root, force=True, recursive=True) self.device.mkdir(self.remote_home_dir, parents=True) self.device.mkdir(self.remote_tmp_dir) self.push_libs() self.push_progs(progs) self.device.chmod(self.remote_bin_dir, recursive=True, root=True) def push_libs(self): if self.options.local_apk: with mozfile.TemporaryDirectory() as tmpdir: apk_contents = ZipFile(self.options.local_apk) for info in apk_contents.infolist(): if info.filename.endswith(".so"): print >> sys.stderr, "Pushing %s.." % info.filename remote_file = posixpath.join( self.remote_bin_dir, os.path.basename(info.filename)) apk_contents.extract(info, tmpdir) local_file = os.path.join(tmpdir, info.filename) with open(local_file) as f: # Decompress xz-compressed file. if f.read(5)[1:] == '7zXZ': cmd = [ 'xz', '-df', '--suffix', '.so', local_file] subprocess.check_output(cmd) # xz strips the ".so" file suffix. os.rename(local_file[:-3], local_file) self.device.push(local_file, remote_file) elif self.options.local_lib: for file in os.listdir(self.options.local_lib): if file.endswith(".so"): print >> sys.stderr, "Pushing %s.." % file remote_file = posixpath.join(self.remote_bin_dir, file) local_file = os.path.join(self.options.local_lib, file) self.device.push(local_file, remote_file) # Additional libraries may be found in a sub-directory such as # "lib/armeabi-v7a" for subdir in ["assets", "lib"]: local_arm_lib = os.path.join(self.options.local_lib, subdir) if os.path.isdir(local_arm_lib): for root, dirs, files in os.walk(local_arm_lib): for file in files: if (file.endswith(".so")): print >> sys.stderr, "Pushing %s.." % file remote_file = posixpath.join( self.remote_bin_dir, file) local_file = os.path.join(root, file) self.device.push(local_file, remote_file) def push_progs(self, progs): for local_file in progs: remote_file = posixpath.join( self.remote_bin_dir, os.path.basename(local_file)) self.device.push(local_file, remote_file) def build_environment(self): env = self.build_core_environment() env['LD_LIBRARY_PATH'] = self.remote_bin_dir env["TMPDIR"] = self.remote_tmp_dir env["HOME"] = self.remote_home_dir env["MOZ_XRE_DIR"] = self.remote_bin_dir if self.options.add_env: for envdef in self.options.add_env: envdef_parts = envdef.split("=", 1) if len(envdef_parts) == 2: env[envdef_parts[0]] = envdef_parts[1] elif len(envdef_parts) == 1: env[envdef_parts[0]] = "" else: self.log.warning( "invalid --addEnv option skipped: %s" % envdef) return env def run_one_test(self, prog, env, symbols_path=None, interactive=False, timeout_factor=1): """ Run a single C++ unit test program remotely. Arguments: * prog: The path to the test program to run. * env: The environment to use for running the program. * symbols_path: A path to a directory containing Breakpad-formatted symbol files for producing stack traces on crash. * timeout_factor: An optional test-specific timeout multiplier. Return True if the program exits with a zero status, False otherwise. """ basename = os.path.basename(prog) remote_bin = posixpath.join(self.remote_bin_dir, basename) self.log.test_start(basename) test_timeout = cppunittests.CPPUnitTests.TEST_PROC_TIMEOUT * \ timeout_factor try: output = self.device.shell_output(remote_bin, env=env, cwd=self.remote_home_dir, timeout=test_timeout) returncode = 0 except ADBProcessError as e: output = e.adb_process.stdout returncode = e.adb_process.exitcode self.log.process_output(basename, "\n%s" % output, command=[remote_bin]) with mozfile.TemporaryDirectory() as tempdir: self.device.pull(self.remote_home_dir, tempdir) if mozcrash.check_for_crashes(tempdir, symbols_path, test_name=basename): self.log.test_end(basename, status='CRASH', expected='PASS') return False result = returncode == 0 if not result: self.log.test_end(basename, status='FAIL', expected='PASS', message=("test failed with return code %s" % returncode)) else: self.log.test_end(basename, status='PASS', expected='PASS') return result
def cli(record, certutil, url, path): # create profile profile = create_profile("firefox") print("Created profile: {}".format(profile.profile)) mitmproxy_home = os.path.join(os.path.expanduser("~"), ".mitmproxy") cert = os.path.join(mitmproxy_home, "mitmproxy-ca-cert.cer") # start mitmdump scripts = os.path.join(os.getcwd(), "scripts") mitmdump = os.path.join(os.getcwd(), "utils", "mitmdump") if record: command = [mitmdump, "--wfile", path] else: command = [ mitmdump, "--replay-kill-extra", "--script", " ".join( [os.path.join(scripts, "alternate-server-replay.py"), path]), ] try: print(command) mitmdump_process = subprocess.Popen(command) # create certificate database certdb = "sql:{}/".format(profile.profile) print("Creating certificate database") command = [certutil, "-N", "-v", "-d", certdb, "--empty-password"] subprocess.call(command) # install mitmproxy certificate command = [ certutil, "-A", "-d", certdb, "-n", "mitmproxy-cert", "-t", "TC,,", "-a", "-i", cert, ] print("Installing {} into certificate database".format(cert)) subprocess.call(command) # verify certificate is installed command = [certutil, "-d", certdb, "-L"] assert "mitmproxy-cert" in subprocess.check_output(command) # setup device device = ADBAndroid() device.create_socket_connection("reverse", "tcp:8080", "tcp:8080") device_storage = "/sdcard/raptor" device_profile = os.path.join(device_storage, "profile") if device.is_dir(device_storage): device.rm(device_storage, recursive=True) device.mkdir(device_storage) device.mkdir(device_profile) userjs = os.path.join(profile.profile, "user.js") with open(userjs) as f: prefs = f.readlines() prefs = [p for p in prefs if "network.proxy" not in p] with open(userjs, "w") as f: f.writelines(prefs) proxy = { "network.proxy.type": 1, "network.proxy.http": "127.0.0.1", "network.proxy.http_port": 8080, "network.proxy.ssl": "127.0.0.1", "network.proxy.ssl_port": 8080, "network.proxy.no_proxies_on": "localhost, 127.0.0.1", } profile.set_preferences(proxy) device.push(profile.profile, device_profile) device.chmod(device_storage, recursive=True) app_args = [ "-profile", device_profile, "--marionette", "--es", "env0", "LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6", ] # start app app_name = "org.mozilla.geckoview_example" activity_name = "GeckoViewActivity" device.stop_application(app_name) device.launch_activity( app_name, activity_name, extra_args=app_args, url=url, e10s=True, fail_if_running=False, ) # wait for mitmdump to finish mitmdump_process.wait() finally: if mitmdump_process is None: mitmdump_process.terminate() exit(mitmdump_process.returncode)
class XPCShellRemote(xpcshell.XPCShellTests, object): def __init__(self, options, log): xpcshell.XPCShellTests.__init__(self, log) self.options = options verbose = False if options['log_tbpl_level'] == 'debug' or options[ 'log_mach_level'] == 'debug': verbose = True self.device = ADBAndroid(adb=options['adbPath'], device=options['deviceSerial'], test_root=options['remoteTestRoot'], verbose=verbose) self.remoteTestRoot = posixpath.join(self.device.test_root, "xpc") # Add Android version (SDK level) to mozinfo so that manifest entries # can be conditional on android_version. mozinfo.info['android_version'] = self.device.version self.localLib = options['localLib'] self.localBin = options['localBin'] self.pathMapping = [] # remoteBinDir contains xpcshell and its wrapper script, both of which must # be executable. Since +x permissions cannot usually be set on /mnt/sdcard, # and the test root may be on /mnt/sdcard, remoteBinDir is set to be on # /data/local, always. self.remoteBinDir = posixpath.join("/data", "local", "xpcb") # Terse directory names are used here ("c" for the components directory) # to minimize the length of the command line used to execute # xpcshell on the remote device. adb has a limit to the number # of characters used in a shell command, and the xpcshell command # line can be quite complex. self.remoteTmpDir = posixpath.join(self.remoteTestRoot, "tmp") self.remoteScriptsDir = self.remoteTestRoot self.remoteComponentsDir = posixpath.join(self.remoteTestRoot, "c") self.remoteModulesDir = posixpath.join(self.remoteTestRoot, "m") self.remoteMinidumpDir = posixpath.join(self.remoteTestRoot, "minidumps") self.remoteClearDirScript = posixpath.join(self.remoteBinDir, "cleardir") self.profileDir = posixpath.join(self.remoteTestRoot, "p") self.remoteDebugger = options['debugger'] self.remoteDebuggerArgs = options['debuggerArgs'] self.testingModulesDir = options['testingModulesDir'] self.env = {} if options['objdir']: self.xpcDir = os.path.join(options['objdir'], "_tests/xpcshell") elif os.path.isdir(os.path.join(here, 'tests')): self.xpcDir = os.path.join(here, 'tests') else: print("Couldn't find local xpcshell test directory", file=sys.stderr) sys.exit(1) if options['localAPK']: self.localAPKContents = ZipFile(options['localAPK']) if options['setup']: self.setupTestDir() self.setupUtilities() self.setupModules() self.setupMinidumpDir() self.remoteAPK = None if options['localAPK']: self.remoteAPK = posixpath.join( self.remoteBinDir, os.path.basename(options['localAPK'])) self.setAppRoot() # data that needs to be passed to the RemoteXPCShellTestThread self.mobileArgs = { 'device': self.device, 'remoteBinDir': self.remoteBinDir, 'remoteScriptsDir': self.remoteScriptsDir, 'remoteComponentsDir': self.remoteComponentsDir, 'remoteModulesDir': self.remoteModulesDir, 'options': self.options, 'remoteDebugger': self.remoteDebugger, 'pathMapping': self.pathMapping, 'profileDir': self.profileDir, 'remoteTmpDir': self.remoteTmpDir, 'remoteMinidumpDir': self.remoteMinidumpDir, 'remoteClearDirScript': self.remoteClearDirScript, } if self.remoteAPK: self.mobileArgs['remoteAPK'] = self.remoteAPK def setLD_LIBRARY_PATH(self): self.env["LD_LIBRARY_PATH"] = self.remoteBinDir def pushWrapper(self): # Rather than executing xpcshell directly, this wrapper script is # used. By setting environment variables and the cwd in the script, # the length of the per-test command line is shortened. This is # often important when using ADB, as there is a limit to the length # of the ADB command line. localWrapper = tempfile.mktemp() f = open(localWrapper, "w") f.write("#!/system/bin/sh\n") for envkey, envval in self.env.iteritems(): f.write("export %s=%s\n" % (envkey, envval)) f.writelines([ "cd $1\n", "echo xpcw: cd $1\n", "shift\n", "echo xpcw: xpcshell \"$@\"\n", "%s/xpcshell \"$@\"\n" % self.remoteBinDir ]) f.close() remoteWrapper = posixpath.join(self.remoteBinDir, "xpcw") self.device.push(localWrapper, remoteWrapper) os.remove(localWrapper) # Removing and re-creating a directory is a common operation which # can be implemented more efficiently with a shell script. localWrapper = tempfile.mktemp() f = open(localWrapper, "w") # The directory may not exist initially, so rm may fail. 'rm -f' is not # supported on some Androids. Similarly, 'test' and 'if [ -d ]' are not # universally available, so we just ignore errors from rm. f.writelines( ["#!/system/bin/sh\n", "rm -r \"$1\"\n", "mkdir \"$1\"\n"]) f.close() self.device.push(localWrapper, self.remoteClearDirScript) os.remove(localWrapper) self.device.chmod(self.remoteBinDir, recursive=True) def buildEnvironment(self): self.buildCoreEnvironment() self.setLD_LIBRARY_PATH() self.env["MOZ_LINKER_CACHE"] = self.remoteBinDir if self.options['localAPK'] and self.appRoot: self.env["GRE_HOME"] = self.appRoot self.env["XPCSHELL_TEST_PROFILE_DIR"] = self.profileDir self.env["TMPDIR"] = self.remoteTmpDir self.env["HOME"] = self.profileDir self.env["XPCSHELL_TEST_TEMP_DIR"] = self.remoteTmpDir self.env["XPCSHELL_MINIDUMP_DIR"] = self.remoteMinidumpDir if self.options['setup']: self.pushWrapper() def setAppRoot(self): # Determine the application root directory associated with the package # name used by the Fennec APK. self.appRoot = None packageName = None if self.options['localAPK']: try: packageName = self.localAPKContents.read("package-name.txt") if packageName: self.appRoot = posixpath.join("/data", "data", packageName.strip()) except Exception as detail: print("unable to determine app root: " + str(detail)) pass return None def setupUtilities(self): self.device.rm(self.remoteTmpDir, force=True, recursive=True) self.device.mkdir(self.remoteTmpDir) self.device.rm(self.remoteBinDir, force=True, recursive=True) remotePrefDir = posixpath.join(self.remoteBinDir, "defaults", "pref") self.device.mkdir(posixpath.join(remotePrefDir, "extra"), parents=True) self.device.mkdir(self.remoteScriptsDir, parents=True) self.device.mkdir(self.remoteComponentsDir, parents=True) local = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'head.js') remoteFile = posixpath.join(self.remoteScriptsDir, "head.js") self.device.push(local, remoteFile) # The xpcshell binary is required for all tests. Additional binaries # are required for some tests. This list should be similar to # TEST_HARNESS_BINS in testing/mochitest/Makefile.in. binaries = [ "xpcshell", "ssltunnel", "certutil", "pk12util", "BadCertServer", "OCSPStaplingServer", "GenerateOCSPResponse", "SymantecSanctionsServer" ] for fname in binaries: local = os.path.join(self.localBin, fname) if os.path.isfile(local): print("Pushing %s.." % fname, file=sys.stderr) remoteFile = posixpath.join(self.remoteBinDir, fname) self.device.push(local, remoteFile) else: print("*** Expected binary %s not found in %s!" % (fname, self.localBin), file=sys.stderr) local = os.path.join(self.localBin, "components/httpd.js") remoteFile = posixpath.join(self.remoteComponentsDir, "httpd.js") self.device.push(local, remoteFile) local = os.path.join(self.localBin, "components/httpd.manifest") remoteFile = posixpath.join(self.remoteComponentsDir, "httpd.manifest") self.device.push(local, remoteFile) if self.options['localAPK']: remoteFile = posixpath.join( self.remoteBinDir, os.path.basename(self.options['localAPK'])) self.device.push(self.options['localAPK'], remoteFile) self.pushLibs() def pushLibs(self): pushed_libs_count = 0 if self.options['localAPK']: try: dir = tempfile.mkdtemp() for info in self.localAPKContents.infolist(): if info.filename.endswith(".so"): print("Pushing %s.." % info.filename, file=sys.stderr) remoteFile = posixpath.join( self.remoteBinDir, os.path.basename(info.filename)) self.localAPKContents.extract(info, dir) localFile = os.path.join(dir, info.filename) with open(localFile) as f: # Decompress xz-compressed file. if f.read(5)[1:] == '7zXZ': cmd = [ 'xz', '-df', '--suffix', '.so', localFile ] subprocess.check_output(cmd) # xz strips the ".so" file suffix. os.rename(localFile[:-3], localFile) self.device.push(localFile, remoteFile) pushed_libs_count += 1 finally: shutil.rmtree(dir) return pushed_libs_count for file in os.listdir(self.localLib): if (file.endswith(".so")): print("Pushing %s.." % file, file=sys.stderr) if 'libxul' in file: print("This is a big file, it could take a while.", file=sys.stderr) localFile = os.path.join(self.localLib, file) remoteFile = posixpath.join(self.remoteBinDir, file) self.device.push(localFile, remoteFile) pushed_libs_count += 1 # Additional libraries may be found in a sub-directory such as "lib/armeabi-v7a" localArmLib = os.path.join(self.localLib, "lib") if os.path.exists(localArmLib): for root, dirs, files in os.walk(localArmLib): for file in files: if (file.endswith(".so")): print("Pushing %s.." % file, file=sys.stderr) localFile = os.path.join(root, file) remoteFile = posixpath.join(self.remoteBinDir, file) self.device.push(localFile, remoteFile) pushed_libs_count += 1 return pushed_libs_count def setupModules(self): if self.testingModulesDir: self.device.push(self.testingModulesDir, self.remoteModulesDir) def setupTestDir(self): print('pushing %s' % self.xpcDir) # The tests directory can be quite large: 5000 files and growing! # Sometimes - like on a low-end aws instance running an emulator - the push # may exceed the default 5 minute timeout, so we increase it here to 10 minutes. self.device.rm(self.remoteTestRoot, force=True, recursive=True) self.device.push(self.xpcDir, self.remoteScriptsDir, timeout=600) def setupMinidumpDir(self): self.device.rm(self.remoteMinidumpDir, force=True, recursive=True) self.device.mkdir(self.remoteMinidumpDir) def buildTestList(self, test_tags=None, test_paths=None, verify=False): xpcshell.XPCShellTests.buildTestList(self, test_tags=test_tags, test_paths=test_paths, verify=verify) uniqueTestPaths = set([]) for test in self.alltests: uniqueTestPaths.add(test['here']) for testdir in uniqueTestPaths: abbrevTestDir = os.path.relpath(testdir, self.xpcDir) remoteScriptDir = posixpath.join(self.remoteScriptsDir, abbrevTestDir) self.pathMapping.append(PathMapping(testdir, remoteScriptDir))
class Raptor(object): """Container class for Raptor""" def __init__(self, app, binary, run_local=False, obj_path=None, gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None, symbols_path=None, host=None, is_release_build=False, debug_mode=False): self.config = {} self.config['app'] = app self.config['binary'] = binary self.config['platform'] = mozinfo.os self.config['processor'] = mozinfo.processor self.config['run_local'] = run_local self.config['obj_path'] = obj_path self.config['gecko_profile'] = gecko_profile self.config['gecko_profile_interval'] = gecko_profile_interval self.config['gecko_profile_entries'] = gecko_profile_entries self.config['symbols_path'] = symbols_path self.config['host'] = host self.config['is_release_build'] = is_release_build self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv') self.log = get_default_logger(component='raptor-main') self.control_server = None self.playback = None self.benchmark = None self.gecko_profiler = None self.post_startup_delay = 30000 self.device = None # debug mode is currently only supported when running locally self.debug_mode = debug_mode if self.config['run_local'] else False # if running debug-mode reduce the pause after browser startup if self.debug_mode: self.post_startup_delay = 3000 self.log.info( "debug-mode enabled, reducing post-browser startup pause to %d ms" % self.post_startup_delay) # Create the profile; for geckoview we want a firefox profile type if self.config['app'] == 'geckoview': self.profile = create_profile('firefox') else: self.profile = create_profile(self.config['app']) # Merge in base profiles with open(os.path.join(self.profile_data_dir, 'profiles.json'), 'r') as fh: base_profiles = json.load(fh)['raptor'] for name in base_profiles: path = os.path.join(self.profile_data_dir, name) self.log.info("Merging profile: {}".format(path)) self.profile.merge(path) # add profile dir to our config self.config['local_profile_dir'] = self.profile.profile # create results holder self.results_handler = RaptorResultsHandler() # when testing desktop browsers we use mozrunner to start the browser; when # testing on android (i.e. geckoview) we use mozdevice to control the device app if self.config['app'] == "geckoview": # create the android device handler; it gets initiated and sets up adb etc self.log.info("creating android device handler using mozdevice") self.device = ADBAndroid(verbose=True) self.device.clear_logcat() else: # create the desktop browser runner self.log.info("creating browser runner using mozrunner") self.output_handler = OutputHandler() process_args = { 'processOutputLine': [self.output_handler], } runner_cls = runners[app] self.runner = runner_cls(binary, profile=self.profile, process_args=process_args) self.log.info("raptor config: %s" % str(self.config)) @property def profile_data_dir(self): if 'MOZ_DEVELOPER_REPO_DIR' in os.environ: return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'profiles') if build: return os.path.join(build.topsrcdir, 'testing', 'profiles') return os.path.join(here, 'profile_data') def start_control_server(self): self.control_server = RaptorControlServer(self.results_handler, self.debug_mode) self.control_server.start() # for android we must make the control server available to the device if self.config['app'] == "geckoview" and self.config['host'] in ( 'localhost', '127.0.0.1'): self.log.info( "making the raptor control server port available to device") _tcp_port = "tcp:%s" % self.control_server.port self.device.create_socket_connection('reverse', _tcp_port, _tcp_port) def get_playback_config(self, test): self.config['playback_tool'] = test.get('playback') self.log.info("test uses playback tool: %s " % self.config['playback_tool']) self.config['playback_binary_manifest'] = test.get( 'playback_binary_manifest', None) _key = 'playback_binary_zip_%s' % self.config['platform'] self.config['playback_binary_zip'] = test.get(_key, None) self.config['playback_pageset_manifest'] = test.get( 'playback_pageset_manifest', None) _key = 'playback_pageset_zip_%s' % self.config['platform'] self.config['playback_pageset_zip'] = test.get(_key, None) self.config['playback_recordings'] = test.get('playback_recordings', None) self.config['python3_win_manifest'] = test.get('python3_win_manifest', None) def run_test(self, test, timeout=None): self.log.info("starting raptor test: %s" % test['name']) self.log.info("test settings: %s" % str(test)) self.log.info("raptor config: %s" % str(self.config)) # benchmark-type tests require the benchmark test to be served out if test.get('type') == "benchmark": self.benchmark = Benchmark(self.config, test) benchmark_port = int(self.benchmark.port) # for android we must make the benchmarks server available to the device if self.config['app'] == "geckoview" and self.config['host'] \ in ('localhost', '127.0.0.1'): self.log.info( "making the raptor benchmarks server port available to device" ) _tcp_port = "tcp:%s" % benchmark_port self.device.create_socket_connection('reverse', _tcp_port, _tcp_port) else: benchmark_port = 0 gen_test_config(self.config['app'], test['name'], self.control_server.port, self.post_startup_delay, host=self.config['host'], b_port=benchmark_port, debug_mode=1 if self.debug_mode else 0) # must intall raptor addon each time because we dynamically update some content # note: for chrome the addon is just a list of paths that ultimately are added # to the chromium command line '--load-extension' argument raptor_webext = os.path.join(webext_dir, 'raptor') self.log.info("installing webext %s" % raptor_webext) self.profile.addons.install(raptor_webext) # add test specific preferences if test.get("preferences", None) is not None: if self.config['app'] == "firefox": self.profile.set_preferences(json.loads(test['preferences'])) else: self.log.info("preferences were configured for the test, \ but we do not install them on non Firefox browsers." ) # on firefox we can get an addon id; chrome addon actually is just cmd line arg if self.config['app'] in ["firefox", "geckoview"]: webext_id = self.profile.addons.addon_details(raptor_webext)['id'] # for android/geckoview, create a top-level raptor folder on the device # sdcard; if it already exists remove it so we start fresh each time if self.config['app'] == "geckoview": self.device_raptor_dir = "/sdcard/raptor" self.config['device_raptor_dir'] = self.device_raptor_dir if self.device.is_dir(self.device_raptor_dir): self.log.info("deleting existing device raptor dir: %s" % self.device_raptor_dir) self.device.rm(self.device_raptor_dir, recursive=True) self.log.info("creating raptor folder on sdcard: %s" % self.device_raptor_dir) self.device.mkdir(self.device_raptor_dir) self.device.chmod(self.device_raptor_dir, recursive=True) # some tests require tools to playback the test pages if test.get('playback', None) is not None: self.get_playback_config(test) # startup the playback tool self.playback = get_playback(self.config, self.device) # for android we must make the playback server available to the device if self.config['app'] == "geckoview" and self.config['host'] \ in ('localhost', '127.0.0.1'): self.log.info( "making the raptor playback server port available to device" ) _tcp_port = "tcp:8080" self.device.create_socket_connection('reverse', _tcp_port, _tcp_port) if self.config['app'] in ("geckoview", "firefox") and \ self.config['host'] not in ('localhost', '127.0.0.1'): # Must delete the proxy settings from the profile if running # the test with a host different from localhost. userjspath = os.path.join(self.profile.profile, 'user.js') with open(userjspath) as userjsfile: prefs = userjsfile.readlines() prefs = [pref for pref in prefs if 'network.proxy' not in pref] with open(userjspath, 'w') as userjsfile: userjsfile.writelines(prefs) # for geckoview/android pageload playback we can't use a policy to turn on the # proxy; we need to set prefs instead; note that the 'host' may be different # than '127.0.0.1' so we must set the prefs accordingly if self.config['app'] == "geckoview" and test.get('playback', None) is not None: self.log.info( "setting profile prefs to turn on the geckoview browser proxy") no_proxies_on = "localhost, 127.0.0.1, %s" % self.config['host'] proxy_prefs = {} proxy_prefs["network.proxy.type"] = 1 proxy_prefs["network.proxy.http"] = self.config['host'] proxy_prefs["network.proxy.http_port"] = 8080 proxy_prefs["network.proxy.ssl"] = self.config['host'] proxy_prefs["network.proxy.ssl_port"] = 8080 proxy_prefs["network.proxy.no_proxies_on"] = no_proxies_on self.profile.set_preferences(proxy_prefs) # now some final settings, and then startup of the browser under test if self.config['app'] == "geckoview": # for android/geckoview we must copy the profile onto the device and set perms if not self.device.is_app_installed(self.config['binary']): raise Exception('%s is not installed' % self.config['binary']) self.device_profile = os.path.join(self.device_raptor_dir, "profile") if self.device.is_dir(self.device_profile): self.log.info("deleting existing device profile folder: %s" % self.device_profile) self.device.rm(self.device_profile, recursive=True) self.log.info("creating profile folder on device: %s" % self.device_profile) self.device.mkdir(self.device_profile) self.log.info("copying firefox profile onto the device") self.log.info("note: the profile folder being copied is: %s" % self.profile.profile) self.log.info( 'the adb push cmd copies that profile dir to a new temp dir before copy' ) self.device.push(self.profile.profile, self.device_profile) self.device.chmod(self.device_profile, recursive=True) # now start the geckoview app self.log.info("starting %s" % self.config['app']) extra_args = [ "-profile", self.device_profile, "--es", "env0", "LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6" ] try: # make sure the geckoview app is not running before # attempting to start. self.device.stop_application(self.config['binary']) self.device.launch_activity(self.config['binary'], "GeckoViewActivity", extra_args=extra_args, url='about:blank', e10s=True, fail_if_running=False) except Exception: self.log.error("Exception launching %s" % self.config['binary']) raise self.control_server.device = self.device self.control_server.app_name = self.config['binary'] else: # For Firefox we need to set # MOZ_DISABLE_NONLOCAL_CONNECTIONS=1 env var before # startup when testing release builds from mozilla-beta or # mozilla-release. This is because of restrictions on # release builds that require webextensions to be signed # unless MOZ_DISABLE_NONLOCAL_CONNECTIONS is set to '1'. if self.config['app'] == "firefox" and self.config[ 'is_release_build']: self.log.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=1") os.environ['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = "1" # if running debug-mode, tell Firefox to open the browser console on startup # for google chrome, open the devtools on the raptor test tab if self.debug_mode: if self.config['app'] == "firefox": self.runner.cmdargs.extend(['-jsconsole']) if self.config['app'] == "chrome": self.runner.cmdargs.extend( ['--auto-open-devtools-for-tabs']) # now start the desktop browser self.log.info("starting %s" % self.config['app']) # if running a pageload test on google chrome, add the cmd line options # to turn on the proxy and ignore security certificate errors # if using host localhost, 127.0.0.1. if self.config['app'] == "chrome" and test.get('playback', None) is not None: chrome_args = [ '--proxy-server="http=127.0.0.1:8080;' + 'https=127.0.0.1:8080;ssl=127.0.0.1:8080"', '--ignore-certificate-errors' ] if self.config['host'] not in ('localhost', '127.0.0.1'): chrome_args[0] = chrome_args[0].replace( '127.0.0.1', self.config['host']) if ' '.join(chrome_args) not in ' '.join(self.runner.cmdargs): self.runner.cmdargs.extend(chrome_args) self.runner.start() proc = self.runner.process_handler self.output_handler.proc = proc self.control_server.browser_proc = proc # pageload tests need to be able to access non-local connections via mitmproxy if self.config['app'] == "firefox" and self.config['is_release_build'] and \ test.get('playback', None) is not None: self.log.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=0") os.environ['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = "0" # if geckoProfile is enabled, initialize it if self.config['gecko_profile'] is True: self._init_gecko_profiling(test) # tell the control server the gecko_profile dir; the control server will # receive the actual gecko profiles from the web ext and will write them # to disk; then profiles are picked up by gecko_profile.symbolicate self.control_server.gecko_profile_dir = self.gecko_profiler.gecko_profile_dir # set our cs flag to indicate we are running the browser/app self.control_server._finished = False # convert to seconds and account for page cycles timeout = int(timeout / 1000) * int(test['page_cycles']) # account for the pause the raptor webext runner takes after browser startup timeout += int(self.post_startup_delay / 1000) # if geckoProfile enabled, give browser more time for profiling if self.config['gecko_profile'] is True: timeout += 5 * 60 try: elapsed_time = 0 while not self.control_server._finished: time.sleep(1) # we only want to force browser-shutdown on timeout if not in debug mode; # in debug-mode we leave the browser running (require manual shutdown) if not self.debug_mode: elapsed_time += 1 if elapsed_time > (timeout) - 5: # stop 5 seconds early self.log.info( "application timed out after {} seconds".format( timeout)) self.control_server.wait_for_quit() break finally: if self.config['app'] != "geckoview": try: self.runner.check_for_crashes() except NotImplementedError: # not implemented for Chrome pass # TODO: if on geckoview is there some cleanup here i.e. check for crashes? if self.playback is not None: self.playback.stop() # remove the raptor webext; as it must be reloaded with each subtest anyway self.log.info("removing webext %s" % raptor_webext) if self.config['app'] in ["firefox", "geckoview"]: self.profile.addons.remove_addon(webext_id) # for chrome the addon is just a list (appended to cmd line) if self.config['app'] in ["chrome", "chrome-android"]: self.profile.addons.remove(raptor_webext) # gecko profiling symbolication if self.config['gecko_profile'] is True: self.gecko_profiler.symbolicate() # clean up the temp gecko profiling folders self.log.info("cleaning up after gecko profiling") self.gecko_profiler.clean() # browser should be closed by now but this is a backup-shutdown (if not in debug-mode) if not self.debug_mode: if self.config['app'] != "geckoview": if self.runner.is_running(): self.runner.stop() # TODO the geckoview app should have been shutdown by this point by the # control server, but we can double-check here to make sure else: # in debug mode, and running locally, leave the browser running if self.config['run_local']: self.log.info( "* debug-mode enabled - please shutdown the browser manually..." ) self.runner.wait(timeout=None) def _init_gecko_profiling(self, test): self.log.info("initializing gecko profiler") upload_dir = os.getenv('MOZ_UPLOAD_DIR') if not upload_dir: self.log.critical( "Profiling ignored because MOZ_UPLOAD_DIR was not set") else: self.gecko_profiler = GeckoProfile(upload_dir, self.config, test) def process_results(self): # when running locally output results in build/raptor.json; when running # in production output to a local.json to be turned into tc job artifact if self.config.get('run_local', False): if 'MOZ_DEVELOPER_REPO_DIR' in os.environ: raptor_json_path = os.path.join( os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'mozharness', 'build', 'raptor.json') else: raptor_json_path = os.path.join(here, 'raptor.json') else: raptor_json_path = os.path.join(os.getcwd(), 'local.json') self.config['raptor_json_path'] = raptor_json_path return self.results_handler.summarize_and_output(self.config) def get_page_timeout_list(self): return self.results_handler.page_timeout_list def clean_up(self): self.control_server.stop() if self.config['app'] != "geckoview": self.runner.stop() elif self.config['app'] == 'geckoview': self.log.info('removing reverse socket connections') self.device.remove_socket_connections('reverse') else: pass self.log.info("finished")
class AbstractAndroidFirefox(object): def __init__(self, certutil, binary): self.proxy = None self.certutil = certutil self.app_args = [ "--marionette", "--es", "env0", "LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6", ] self.binary = binary self.skip_install = False self.profile = None def set_profile(self): self.profile = create_profile("firefox") print("Created profile: {}".format(self.profile.profile)) def create_certificate(self): certdb = "sql:{}/".format(self.profile.profile) print("Creating certificate database") command = [self.certutil, "-N", "-v", "-d", certdb, "--empty-password"] subprocess.call(command) # install mitmproxy certificate command = [ self.certutil, "-A", "-d", certdb, "-n", "mitmproxy-cert", "-t", "TC,,", "-a", "-i", self.proxy.cert, ] print("Installing {} into certificate database".format( self.proxy.cert)) subprocess.call(command) command = [self.certutil, "-d", certdb, "-L"] assert "mitmproxy-cert" in subprocess.check_output(command) def setup_device(self): self.device = ADBAndroid() if self.binary and self.proxy.mode is "record": if not self.skip_install: if self.device.is_app_installed(self.APP_NAME): print("Uninstalling app %s" % self.APP_NAME) self.device.uninstall_app(self.APP_NAME) print("Installing app %s" % self.APP_NAME) self.device.install_app(apk_path=self.binary) self.skip_install = True else: print("App already installed in a previous recording!!!!") else: print( "No binary provided or proxy in replay mode! Using existing app on the device." ) def setup_app(self): self.device.shell("pm clear {}".format(self.APP_NAME)) self.device.create_socket_connection("reverse", "tcp:8080", "tcp:8080") device_storage = "/sdcard/raptor" device_profile = os.path.join(device_storage, "profile") if self.device.is_dir(device_storage): self.device.rm(device_storage, recursive=True) self.device.mkdir(device_storage) self.device.mkdir(device_profile) self.app_args.extend(["-profile", device_profile]) userjs = os.path.join(self.profile.profile, "user.js") with open(userjs) as f: prefs = f.readlines() prefs = [p for p in prefs if "network.proxy" not in p] with open(userjs, "w") as f: f.writelines(prefs) self.profile.set_preferences({ "network.proxy.type": 1, "network.proxy.http": "127.0.0.1", "network.proxy.http_port": 8080, "network.proxy.ssl": "127.0.0.1", "network.proxy.ssl_port": 8080, "network.proxy.no_proxies_on": "localhost, 127.0.0.1", }) self.device.push(self.profile.profile, device_profile) self.device.chmod(device_storage, recursive=True) def run_android_app(self, url): raise NotImplementedError def start(self, url="about:blank", proxy_service=None): self.proxy = proxy_service self.setup_device() self.set_profile() self.create_certificate() self.setup_app() self.run_android_app(url) def stop(self): self.device.stop_application(self.APP_NAME) def screen_shot(self, path): try: self.device.rm("/sdcard/screen.png") except ADBProcessError as e: pass self.device.shell("screencap -p /sdcard/screen.png") self.device.pull("/sdcard/screen.png", path) self.device.rm("/sdcard/screen.png") def app_information(self): if self.binary: return mozversion.get_version(binary=self.binary) return None
class MobileApp(): def __init__(self, config): self.config = config self.log = get_default_logger(component='recorder-mobile-app - ') self.profile = self.config['profile'] def setup_app(self): # create the android device handler; it gets initiated and sets up adb etc self.log.info("creating android device handler using mozdevice") self.device = ADBAndroid(verbose=True, logger_name="recorder-adb - ") self.device_raptor_dir = "/sdcard/raptor" self.config['device_raptor_dir'] = self.device_raptor_dir if self.device.is_dir(self.device_raptor_dir): self.log.info("deleting existing device raptor dir: %s" % self.device_raptor_dir) self.device.rm(self.device_raptor_dir, recursive=True) self.log.info("creating raptor folder on sdcard: %s" % self.device_raptor_dir) self.device.mkdir(self.device_raptor_dir) self.device.chmod(self.device_raptor_dir, recursive=True) def start_app(self): self.log.info("Adb Port redirect:") _tcp_port = "tcp:8080" self.device.create_socket_connection('reverse', _tcp_port, _tcp_port) userjspath = os.path.join(self.profile.profile, 'user.js') with open(userjspath) as userjsfile: prefs = userjsfile.readlines() prefs = [pref for pref in prefs if 'network.proxy' not in pref] with open(userjspath, 'w') as userjsfile: userjsfile.writelines(prefs) no_proxies_on = "localhost, 127.0.0.1, %s" % self.config['host'] proxy_prefs = {} proxy_prefs["network.proxy.type"] = 1 proxy_prefs["network.proxy.http"] = self.config['host'] proxy_prefs["network.proxy.http_port"] = 8080 proxy_prefs["network.proxy.ssl"] = self.config['host'] proxy_prefs["network.proxy.ssl_port"] = 8080 proxy_prefs["network.proxy.no_proxies_on"] = no_proxies_on self.profile.set_preferences(proxy_prefs) self.device_profile = os.path.join(self.device_raptor_dir, "profile") if self.device.is_dir(self.device_profile): self.log.info("deleting existing device profile folder: %s" % self.device_profile) self.device.rm(self.device_profile, recursive=True) self.log.info("creating profile folder on device: %s" % self.device_profile) self.device.mkdir(self.device_profile) self.log.info("copying firefox profile onto the device") self.log.info("note: the profile folder being copied is: %s" % self.profile.profile) self.log.info( 'the adb push cmd copies that profile dir to a new temp dir before copy' ) self.device.push(self.profile.profile, self.device_profile) self.device.chmod(self.device_profile, recursive=True) # now start the geckoview/fennec app self.log.info("starting %s" % self.config['app']) extra_args = [ "-profile", self.device_profile, "--marionette", "--es", "env0", "LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6", ] # launch geckoview example app try: # make sure the geckoview app is not running before # attempting to start. self.device.stop_application(self.config['binary']) self.device.launch_activity(self.config['binary'], "GeckoViewActivity", extra_args=extra_args, url='about:blank', e10s=True, fail_if_running=False) except Exception: self.log.error("Exception launching %s" % self.config['binary']) raise
class RobocopTestRunner(MochitestDesktop): """ A test harness for Robocop. Robocop tests are UI tests for Firefox for Android, based on the Robotium test framework. This harness leverages some functionality from mochitest, for convenience. """ # Some robocop tests run for >60 seconds without generating any output. NO_OUTPUT_TIMEOUT = 180 def __init__(self, options, message_logger): """ Simple one-time initialization. """ MochitestDesktop.__init__(self, options.flavor, vars(options)) verbose = False if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug': verbose = True self.device = ADBAndroid(adb=options.adbPath or 'adb', device=options.deviceSerial, test_root=options.remoteTestRoot, verbose=verbose) # Check that Firefox is installed expected = options.app.split('/')[-1] if not self.device.is_app_installed(expected): raise Exception("%s is not installed on this device" % expected) options.logFile = "robocop.log" if options.remoteTestRoot is None: options.remoteTestRoot = self.device.test_root self.remoteProfile = posixpath.join(options.remoteTestRoot, "profile") self.remoteProfileCopy = posixpath.join(options.remoteTestRoot, "profile-copy") self.remoteModulesDir = posixpath.join(options.remoteTestRoot, "modules/") self.remoteConfigFile = posixpath.join(options.remoteTestRoot, "robotium.config") self.remoteLogFile = posixpath.join(options.remoteTestRoot, "logs", "robocop.log") self.options = options process_args = {'messageLogger': message_logger} self.auto = RemoteAutomation(self.device, options.remoteappname, self.remoteProfile, self.remoteLogFile, processArgs=process_args) self.environment = self.auto.environment self.remoteScreenshots = "/mnt/sdcard/Robotium-Screenshots" self.remoteMozLog = posixpath.join(options.remoteTestRoot, "mozlog") self.localLog = options.logFile self.localProfile = None self.certdbNew = True self.passed = 0 self.failed = 0 self.todo = 0 def startup(self): """ Second-stage initialization: One-time initialization which may require cleanup. """ # Despite our efforts to clean up servers started by this script, in practice # we still see infrequent cases where a process is orphaned and interferes # with future tests, typically because the old server is keeping the port in use. # Try to avoid those failures by checking for and killing servers before # trying to start new ones. self.killNamedProc('ssltunnel') self.killNamedProc('xpcshell') self.auto.deleteANRs() self.auto.deleteTombstones() procName = self.options.app.split('/')[-1] self.device.stop_application(procName) if self.device.process_exist(procName): self.log.warning("unable to kill %s before running tests!" % procName) self.device.rm(self.remoteScreenshots, force=True, recursive=True) self.device.rm(self.remoteMozLog, force=True, recursive=True) self.device.mkdir(self.remoteMozLog) logParent = posixpath.dirname(self.remoteLogFile) self.device.rm(logParent, force=True, recursive=True) self.device.mkdir(logParent) # Add Android version (SDK level) to mozinfo so that manifest entries # can be conditional on android_version. self.log.info( "Android sdk version '%s'; will use this to filter manifests" % str(self.device.version)) mozinfo.info['android_version'] = str(self.device.version) if self.options.robocopApk: self.device.install_app(self.options.robocopApk, replace=True) self.log.debug("Robocop APK %s installed" % self.options.robocopApk) # Display remote diagnostics; if running in mach, keep output terse. if self.options.log_mach is None: self.printDeviceInfo() self.setupLocalPaths() self.buildProfile() # ignoreSSLTunnelExts is a workaround for bug 1109310 self.startServers(self.options, debuggerInfo=None, ignoreSSLTunnelExts=True) self.log.debug("Servers started") def cleanup(self): """ Cleanup at end of job run. """ self.log.debug("Cleaning up...") self.stopServers() self.device.stop_application(self.options.app.split('/')[-1]) uploadDir = os.environ.get('MOZ_UPLOAD_DIR', None) if uploadDir: self.log.debug( "Pulling any remote moz logs and screenshots to %s." % uploadDir) if self.device.is_dir(self.remoteMozLog): self.device.pull(self.remoteMozLog, uploadDir) if self.device.is_dir(self.remoteScreenshots): self.device.pull(self.remoteScreenshots, uploadDir) MochitestDesktop.cleanup(self, self.options) if self.localProfile: mozfile.remove(self.localProfile) self.device.rm(self.remoteProfile, force=True, recursive=True) self.device.rm(self.remoteProfileCopy, force=True, recursive=True) self.device.rm(self.remoteScreenshots, force=True, recursive=True) self.device.rm(self.remoteMozLog, force=True, recursive=True) self.device.rm(self.remoteConfigFile, force=True) self.device.rm(self.remoteLogFile, force=True) self.log.debug("Cleanup complete.") def findPath(self, paths, filename=None): for path in paths: p = path if filename: p = os.path.join(p, filename) if os.path.exists(self.getFullPath(p)): return path return None def makeLocalAutomation(self): localAutomation = Automation() localAutomation.IS_WIN32 = False localAutomation.IS_LINUX = False localAutomation.IS_MAC = False localAutomation.UNIXISH = False hostos = sys.platform if (hostos == 'mac' or hostos == 'darwin'): localAutomation.IS_MAC = True elif (hostos == 'linux' or hostos == 'linux2'): localAutomation.IS_LINUX = True localAutomation.UNIXISH = True elif (hostos == 'win32' or hostos == 'win64'): localAutomation.BIN_SUFFIX = ".exe" localAutomation.IS_WIN32 = True return localAutomation def setupLocalPaths(self): """ Setup xrePath and utilityPath and verify xpcshell. This is similar to switchToLocalPaths in runtestsremote.py. """ localAutomation = self.makeLocalAutomation() paths = [self.options.xrePath, localAutomation.DIST_BIN] self.options.xrePath = self.findPath(paths) if self.options.xrePath is None: self.log.error( "unable to find xulrunner path for %s, please specify with --xre-path" % os.name) sys.exit(1) self.log.debug("using xre path %s" % self.options.xrePath) xpcshell = "xpcshell" if (os.name == "nt"): xpcshell += ".exe" if self.options.utilityPath: paths = [self.options.utilityPath, self.options.xrePath] else: paths = [self.options.xrePath] self.options.utilityPath = self.findPath(paths, xpcshell) if self.options.utilityPath is None: self.log.error( "unable to find utility path for %s, please specify with --utility-path" % os.name) sys.exit(1) self.log.debug("using utility path %s" % self.options.utilityPath) xpcshell_path = os.path.join(self.options.utilityPath, xpcshell) if localAutomation.elf_arm(xpcshell_path): self.log.error('xpcshell at %s is an ARM binary; please use ' 'the --utility-path argument to specify the path ' 'to a desktop version.' % xpcshell_path) sys.exit(1) self.log.debug("xpcshell found at %s" % xpcshell_path) def buildProfile(self): """ Build a profile locally, keep it locally for use by servers and push a copy to the remote profile-copy directory. This is similar to buildProfile in runtestsremote.py. """ self.options.extraPrefs.append('browser.search.suggest.enabled=true') self.options.extraPrefs.append('browser.search.suggest.prompted=true') self.options.extraPrefs.append('layout.css.devPixelsPerPx=1.0') self.options.extraPrefs.append('browser.chrome.dynamictoolbar=false') self.options.extraPrefs.append('browser.snippets.enabled=false') self.options.extraPrefs.append('extensions.autoupdate.enabled=false') # Override the telemetry init delay for integration testing. self.options.extraPrefs.append('toolkit.telemetry.initDelay=1') self.options.extensionsToExclude.extend([ '*****@*****.**', ]) self.extraPrefs = parse_preferences(self.options.extraPrefs) if self.options.testingModulesDir: try: self.device.push(self.options.testingModulesDir, self.remoteModulesDir) self.device.chmod(self.remoteModulesDir, recursive=True, root=True) except Exception: self.log.error( "Automation Error: Unable to copy test modules to device.") raise savedTestingModulesDir = self.options.testingModulesDir self.options.testingModulesDir = self.remoteModulesDir else: savedTestingModulesDir = None manifest = MochitestDesktop.buildProfile(self, self.options) if savedTestingModulesDir: self.options.testingModulesDir = savedTestingModulesDir self.localProfile = self.options.profilePath self.log.debug("Profile created at %s" % self.localProfile) # some files are not needed for robocop; save time by not pushing os.remove(os.path.join(self.localProfile, 'userChrome.css')) try: self.device.push(self.localProfile, self.remoteProfileCopy) except Exception: self.log.error( "Automation Error: Unable to copy profile to device.") raise return manifest def setupRemoteProfile(self): """ Remove any remote profile and re-create it. """ self.log.debug("Updating remote profile at %s" % self.remoteProfile) self.device.rm(self.remoteProfile, force=True, recursive=True) self.device.cp(self.remoteProfileCopy, self.remoteProfile, recursive=True) def parseLocalLog(self): """ Read and parse the local log file, noting any failures. """ with open(self.localLog) as currentLog: data = currentLog.readlines() os.unlink(self.localLog) start_found = False end_found = False fail_found = False for line in data: try: message = json.loads(line) if not isinstance(message, dict) or 'action' not in message: continue except ValueError: continue if message['action'] == 'test_end': end_found = True start_found = False break if start_found and not end_found: if 'status' in message: if 'expected' in message: self.failed += 1 elif message['status'] == 'PASS': self.passed += 1 elif message['status'] == 'FAIL': self.todo += 1 if message['action'] == 'test_start': start_found = True if 'expected' in message: fail_found = True result = 0 if fail_found: result = 1 if not end_found: self.log.info( "PROCESS-CRASH | Automation Error: Missing end of test marker (process crashed?)" ) result = 1 return result def logTestSummary(self): """ Print a summary of all tests run to stdout, for treeherder parsing (logging via self.log does not work here). """ print("0 INFO TEST-START | Shutdown") print("1 INFO Passed: %s" % (self.passed)) print("2 INFO Failed: %s" % (self.failed)) print("3 INFO Todo: %s" % (self.todo)) print("4 INFO SimpleTest FINISHED") if self.failed > 0: return 1 return 0 def printDeviceInfo(self, printLogcat=False): """ Log remote device information and logcat (if requested). This is similar to printDeviceInfo in runtestsremote.py """ try: if printLogcat: logcat = self.device.get_logcat( filter_out_regexps=fennecLogcatFilters) for l in logcat: self.log.info(l.decode('utf-8', 'replace')) self.log.info("Device info:") devinfo = self.device.get_info() for category in devinfo: if type(devinfo[category]) is list: self.log.info(" %s:" % category) for item in devinfo[category]: self.log.info(" %s" % item) else: self.log.info(" %s: %s" % (category, devinfo[category])) self.log.info("Test root: %s" % self.device.test_root) except ADBTimeoutError: raise except Exception as e: self.log.warning("Error getting device information: %s" % str(e)) def setupRobotiumConfig(self, browserEnv): """ Create robotium.config and push it to the device. """ fHandle = tempfile.NamedTemporaryFile(suffix='.config', prefix='robotium-', dir=os.getcwd(), delete=False) fHandle.write("profile=%s\n" % self.remoteProfile) fHandle.write("logfile=%s\n" % self.remoteLogFile) fHandle.write("host=http://mochi.test:8888/tests\n") fHandle.write("rawhost=http://%s:%s/tests\n" % (self.options.remoteWebServer, self.options.httpPort)) if browserEnv: envstr = "" delim = "" for key, value in browserEnv.items(): try: value.index(',') self.log.error( "setupRobotiumConfig: browserEnv - Found a ',' " "in our value, unable to process value. key=%s,value=%s" % (key, value)) self.log.error("browserEnv=%s" % browserEnv) except ValueError: envstr += "%s%s=%s" % (delim, key, value) delim = "," fHandle.write("envvars=%s\n" % envstr) fHandle.close() self.device.rm(self.remoteConfigFile, force=True) self.device.push(fHandle.name, self.remoteConfigFile) os.unlink(fHandle.name) def buildBrowserEnv(self): """ Return an environment dictionary suitable for remote use. This is similar to buildBrowserEnv in runtestsremote.py. """ browserEnv = self.environment(xrePath=None, debugger=None) # remove desktop environment not used on device if "XPCOM_MEM_BLOAT_LOG" in browserEnv: del browserEnv["XPCOM_MEM_BLOAT_LOG"] browserEnv["MOZ_LOG_FILE"] = os.path.join(self.remoteMozLog, self.mozLogName) try: browserEnv.update( dict( parse_key_value(self.options.environment, context='--setenv'))) except KeyValueParseError as e: self.log.error(str(e)) return None return browserEnv def runSingleTest(self, test): """ Run the specified test. """ self.log.debug("Running test %s" % test['name']) self.mozLogName = "moz-%s.log" % test['name'] browserEnv = self.buildBrowserEnv() self.setupRobotiumConfig(browserEnv) self.setupRemoteProfile() self.options.app = "am" timeout = None testName = test['name'].split('/')[-1].split('.java')[0] if self.options.enable_coverage: remoteCoverageFile = posixpath.join( self.options.remoteTestRoot, 'robocop-coverage-%s.ec' % testName) coverageFile = os.path.join(self.options.coverage_output_dir, 'robocop-coverage-%s.ec' % testName) if self.options.autorun: # This launches a test (using "am instrument") and instructs # Fennec to /quit/ the browser (using Robocop:Quit) and to # /finish/ all opened activities. browserArgs = [ "instrument", ] if self.options.enable_coverage: browserArgs += [ "-e", "coverage", "true", "-e", "coverageFile", remoteCoverageFile, ] browserArgs += [ "-e", "quit_and_finish", "1", "-e", "deviceroot", self.device.test_root, "-e", "class", "org.mozilla.gecko.tests.%s" % testName, "org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner", ] else: # This does not launch a test at all. It launches an activity # that starts Fennec and then waits indefinitely, since cat # never returns. browserArgs = [ "start", "-n", "org.mozilla.roboexample.test/org.mozilla." "gecko.LaunchFennecWithConfigurationActivity", "&&", "cat" ] timeout = sys.maxint # Forever. self.log.info("") self.log.info( "Serving mochi.test Robocop root at http://%s:%s/tests/robocop/" % (self.options.remoteWebServer, self.options.httpPort)) self.log.info("") result = -1 log_result = -1 try: self.device.clear_logcat() if not timeout: timeout = self.options.timeout if not timeout: timeout = self.NO_OUTPUT_TIMEOUT result, _ = self.auto.runApp(None, browserEnv, "am", self.localProfile, browserArgs, timeout=timeout, symbolsPath=self.options.symbolsPath) self.log.debug("runApp completes with status %d" % result) if result != 0: self.log.error("runApp() exited with code %s" % result) if self.device.is_file(self.remoteLogFile): self.device.pull(self.remoteLogFile, self.localLog) self.device.rm(self.remoteLogFile) log_result = self.parseLocalLog() if result != 0 or log_result != 0: # Display remote diagnostics; if running in mach, keep output # terse. if self.options.log_mach is None: self.printDeviceInfo(printLogcat=True) if self.options.enable_coverage: if self.device.is_file(remoteCoverageFile): self.device.pull(remoteCoverageFile, coverageFile) self.device.rm(remoteCoverageFile) else: self.log.warning( "Code coverage output not found on remote device: %s" % remoteCoverageFile) except Exception: self.log.error( "Automation Error: Exception caught while running tests") traceback.print_exc() result = 1 self.log.debug("Test %s completes with status %d (log status %d)" % (test['name'], int(result), int(log_result))) return result def runTests(self): self.startup() if isinstance(self.options.manifestFile, TestManifest): mp = self.options.manifestFile else: mp = TestManifest(strict=False) mp.read("robocop.ini") filters = [] if self.options.totalChunks: filters.append( chunk_by_slice(self.options.thisChunk, self.options.totalChunks)) robocop_tests = mp.active_tests(exists=False, filters=filters, **mozinfo.info) if not self.options.autorun: # Force a single loop iteration. The iteration will start Fennec and # the httpd server, but not actually run a test. self.options.test_paths = [robocop_tests[0]['name']] active_tests = [] for test in robocop_tests: if self.options.test_paths and test[ 'name'] not in self.options.test_paths: continue if 'disabled' in test: self.log.info('TEST-INFO | skipping %s | %s' % (test['name'], test['disabled'])) continue active_tests.append(test) tests_by_manifest = defaultdict(list) for test in active_tests: tests_by_manifest[test['manifest']].append(test['name']) self.log.suite_start(tests_by_manifest) worstTestResult = None for test in active_tests: result = self.runSingleTest(test) if worstTestResult is None or worstTestResult == 0: worstTestResult = result if worstTestResult is None: self.log.warning( "No tests run. Did you pass an invalid TEST_PATH?") worstTestResult = 1 else: print "INFO | runtests.py | Test summary: start." logResult = self.logTestSummary() print "INFO | runtests.py | Test summary: end." if worstTestResult == 0: worstTestResult = logResult return worstTestResult
def start(self, url="about:blank"): # create profile profile = create_profile("firefox") print("Created profile: {}".format(profile.profile)) # create certificate database certdb = "sql:{}/".format(profile.profile) print("Creating certificate database") command = [self.certutil, "-N", "-v", "-d", certdb, "--empty-password"] subprocess.call(command) # install mitmproxy certificate command = [ self.certutil, "-A", "-d", certdb, "-n", "mitmproxy-cert", "-t", "TC,,", "-a", "-i", self.proxy.cert, ] print("Installing {} into certificate database".format( self.proxy.cert)) subprocess.call(command) # verify certificate is installed command = [self.certutil, "-d", certdb, "-L"] assert "mitmproxy-cert" in subprocess.check_output(command) # setup device device = ADBAndroid() device.shell("pm clear {}".format(self.APP_NAME)) device.create_socket_connection("reverse", "tcp:8080", "tcp:8080") device_storage = "/sdcard/raptor" device_profile = os.path.join(device_storage, "profile") if device.is_dir(device_storage): device.rm(device_storage, recursive=True) device.mkdir(device_storage) device.mkdir(device_profile) userjs = os.path.join(profile.profile, "user.js") with open(userjs) as f: prefs = f.readlines() prefs = [p for p in prefs if "network.proxy" not in p] with open(userjs, "w") as f: f.writelines(prefs) profile.set_preferences({ "network.proxy.type": 1, "network.proxy.http": "127.0.0.1", "network.proxy.http_port": 8080, "network.proxy.ssl": "127.0.0.1", "network.proxy.ssl_port": 8080, "network.proxy.no_proxies_on": "localhost, 127.0.0.1", }) device.push(profile.profile, device_profile) device.chmod(device_storage, recursive=True) app_args = [ "-profile", device_profile, "--marionette", "--es", "env0", "LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6", ] # start app device.stop_application(self.APP_NAME) device.launch_activity( self.APP_NAME, self.ACTIVITY_NAME, extra_args=app_args, url=url, e10s=True, fail_if_running=False, )