def run_test_harness(options): if options is None: raise ValueError( "Invalid options specified, use --help for a list of valid options") message_logger = MessageLogger(logger=None) process_args = {'messageLogger': message_logger} auto = RemoteAutomation(None, "fennec", processArgs=process_args) auto.setDeviceManager(options.dm) runResult = -1 robocop = RobocopTestRunner(auto, options.dm, options) try: message_logger.logger = robocop.log message_logger.buffering = False robocop.message_logger = message_logger robocop.log.debug("options=%s" % vars(options)) runResult = robocop.runTests() except KeyboardInterrupt: robocop.log.info("runrobocop.py | Received keyboard interrupt") runResult = -1 except: traceback.print_exc() robocop.log.error( "runrobocop.py | Received unexpected exception while running tests") runResult = 1 finally: try: robocop.cleanup() except devicemanager.DMError: # ignore device error while cleaning up pass message_logger.finish() return runResult
def main(): dm = DeviceManager(None, None) automation = RemoteAutomation(dm) parser = RemoteOptions(automation) options, args = parser.parse_args() if (options.deviceIP == None): print "Error: you must provide a device IP to connect to via the --device option" sys.exit(1) dm = DeviceManager(options.deviceIP, options.devicePort) automation.setDeviceManager(dm) if (options.remoteProductName != None): automation.setProduct(options.remoteProductName) # Set up the defaults and ensure options are set options = parser.verifyRemoteOptions(options) if (options == None): print "ERROR: Invalid options specified, use --help for a list of valid options" sys.exit(1) automation.setAppName(options.app) automation.setRemoteProfile(options.remoteProfile) reftest = RemoteReftest(automation, dm, options, SCRIPT_DIRECTORY) # Start the webserver reftest.startWebServer(options) #an example manifest name to use on the cli # manifest = "http://" + options.remoteWebServer + "/reftests/layout/reftests/reftest-sanity/reftest.list" reftest.runTests(args[0], options) reftest.stopWebServer(options)
def run_test_harness(options): if options is None: raise ValueError( "Invalid options specified, use --help for a list of valid options" ) message_logger = MessageLogger(logger=None) process_args = {'messageLogger': message_logger} auto = RemoteAutomation(None, "fennec", processArgs=process_args) auto.setDeviceManager(options.dm) runResult = -1 robocop = RobocopTestRunner(auto, options.dm, options) try: message_logger.logger = robocop.log message_logger.buffering = False robocop.message_logger = message_logger robocop.log.debug("options=%s" % vars(options)) runResult = robocop.runTests() except KeyboardInterrupt: robocop.log.info("runrobocop.py | Received keyboard interrupt") runResult = -1 except: traceback.print_exc() robocop.log.error( "runrobocop.py | Received unexpected exception while running tests" ) runResult = 1 finally: try: robocop.cleanup() except devicemanager.DMError: # ignore device error while cleaning up pass message_logger.finish() return runResult
def main(): scriptdir = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) dm = devicemanager.DeviceManager(None, None) auto = RemoteAutomation(dm, "fennec") parser = RemoteOptions(auto, scriptdir) options, args = parser.parse_args() dm = devicemanager.DeviceManager(options.deviceIP, options.devicePort) auto.setDeviceManager(dm) options = parser.verifyRemoteOptions(options, auto) if (options == None): print "ERROR: Invalid options specified, use --help for a list of valid options" sys.exit(1) productPieces = options.remoteProductName.split('.') if (productPieces != None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) mochitest = MochiRemote(auto, dm, options) options = parser.verifyOptions(options, mochitest) if (options == None): sys.exit(1) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) sys.exit(mochitest.runTests(options))
def main(): dm = DeviceManager(None, None) automation = RemoteAutomation(dm) parser = RemoteOptions(automation) options, args = parser.parse_args() if options.deviceIP == None: print "Error: you must provide a device IP to connect to via the --device option" sys.exit(1) dm = DeviceManager(options.deviceIP, options.devicePort) automation.setDeviceManager(dm) if options.remoteProductName != None: automation.setProduct(options.remoteProductName) # Set up the defaults and ensure options are set options = parser.verifyRemoteOptions(options) if options == None: print "ERROR: Invalid options specified, use --help for a list of valid options" sys.exit(1) automation.setAppName(options.app) automation.setRemoteProfile(options.remoteProfile) reftest = RemoteReftest(automation, dm, options, SCRIPT_DIRECTORY) # Start the webserver reftest.startWebServer(options) # an example manifest name to use on the cli # manifest = "http://" + options.remoteWebServer + "/reftests/layout/reftests/reftest-sanity/reftest.list" reftest.runTests(args[0], options) reftest.stopWebServer(options)
def __init__(self, options, message_logger): """ Simple one-time initialization. """ MochitestDesktop.__init__(self, options.flavor, vars(options)) verbose = False if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug': verbose = True self.device = ADBDevice(adb=options.adbPath or 'adb', device=options.deviceSerial, test_root=options.remoteTestRoot, verbose=verbose) # Check that Firefox is installed expected = options.app.split('/')[-1] if not self.device.is_app_installed(expected): raise Exception("%s is not installed on this device" % expected) options.logFile = "robocop.log" if options.remoteTestRoot is None: options.remoteTestRoot = self.device.test_root self.remoteProfile = posixpath.join(options.remoteTestRoot, "profile") self.remoteProfileCopy = posixpath.join(options.remoteTestRoot, "profile-copy") self.remoteModulesDir = posixpath.join(options.remoteTestRoot, "modules/") self.remoteConfigFile = posixpath.join(options.remoteTestRoot, "robotium.config") self.remoteLogFile = posixpath.join(options.remoteTestRoot, "logs", "robocop.log") self.options = options process_args = {'messageLogger': message_logger} self.auto = RemoteAutomation(self.device, options.remoteappname, self.remoteProfile, self.remoteLogFile, processArgs=process_args) self.environment = self.auto.environment self.remoteScreenshots = "/mnt/sdcard/Robotium-Screenshots" self.remoteMozLog = posixpath.join(options.remoteTestRoot, "mozlog") self.localLog = options.logFile self.localProfile = None self.certdbNew = True self.passed = 0 self.failed = 0 self.todo = 0
def run_test_harness(parser, options): parser.validate(options) if options is None: raise ValueError( "Invalid options specified, use --help for a list of valid options" ) message_logger = MessageLogger(logger=None) process_args = {'messageLogger': message_logger} auto = RemoteAutomation(None, "fennec", processArgs=process_args) auto.setDeviceManager(options.dm) runResult = -1 robocop = RobocopTestRunner(auto, options.dm, options) # Check that Firefox is installed expected = options.app.split('/')[-1] installed = options.dm.shellCheckOutput( ['pm', 'list', 'packages', expected]) if expected not in installed: robocop.log.error("%s is not installed on this device" % expected) return 1 try: message_logger.logger = robocop.log message_logger.buffering = False robocop.message_logger = message_logger robocop.log.debug("options=%s" % vars(options)) runResult = robocop.runTests() except KeyboardInterrupt: robocop.log.info("runrobocop.py | Received keyboard interrupt") runResult = -1 except: traceback.print_exc() robocop.log.error( "runrobocop.py | Received unexpected exception while running tests" ) runResult = 1 finally: try: robocop.cleanup() except mozdevice.DMError: # ignore device error while cleaning up pass message_logger.finish() return runResult
def switchToLocalPaths(self, options): """ Set local paths in the options, return a function that will restore remote values """ remoteXrePath = options.xrePath remoteProfilePath = options.profilePath remoteUtilityPath = options.utilityPath paths = [ options.xrePath, ] if build_obj: paths.append(os.path.join(build_obj.topobjdir, "dist", "bin")) options.xrePath = self.findPath(paths) if options.xrePath is None: self.log.error( "unable to find xulrunner path for %s, please specify with --xre-path" % os.name ) sys.exit(1) xpcshell = "xpcshell" if os.name == "nt": xpcshell += ".exe" if options.utilityPath: paths = [options.utilityPath, options.xrePath] else: paths = [options.xrePath] options.utilityPath = self.findPath(paths, xpcshell) if options.utilityPath is None: self.log.error( "unable to find utility path for %s, please specify with --utility-path" % os.name ) sys.exit(1) xpcshell_path = os.path.join(options.utilityPath, xpcshell) if RemoteAutomation.elf_arm(xpcshell_path): self.log.error( "xpcshell at %s is an ARM binary; please use " "the --utility-path argument to specify the path " "to a desktop version." % xpcshell_path ) sys.exit(1) if self.localProfile: options.profilePath = self.localProfile else: options.profilePath = None def fixup(): options.xrePath = remoteXrePath options.utilityPath = remoteUtilityPath options.profilePath = remoteProfilePath return fixup
def run_test_harness(parser, options): parser.validate(options) if options is None: raise ValueError( "Invalid options specified, use --help for a list of valid options") message_logger = MessageLogger(logger=None) process_args = {'messageLogger': message_logger} auto = RemoteAutomation(None, "fennec", processArgs=process_args) auto.setDeviceManager(options.dm) runResult = -1 robocop = RobocopTestRunner(auto, options.dm, options) # Check that Firefox is installed expected = options.app.split('/')[-1] installed = options.dm.shellCheckOutput(['pm', 'list', 'packages', expected]) if expected not in installed: robocop.log.error("%s is not installed on this device" % expected) return 1 try: message_logger.logger = robocop.log message_logger.buffering = False robocop.message_logger = message_logger robocop.log.debug("options=%s" % vars(options)) runResult = robocop.runTests() except KeyboardInterrupt: robocop.log.info("runrobocop.py | Received keyboard interrupt") runResult = -1 except: traceback.print_exc() robocop.log.error( "runrobocop.py | Received unexpected exception while running tests") runResult = 1 finally: try: robocop.cleanup() except mozdevice.DMError: # ignore device error while cleaning up pass message_logger.finish() return runResult
def start(self): "Run the Refest server, returning the process ID of the server." env = dict(os.environ) env["XPCOM_DEBUG_BREAK"] = "warn" bin_suffix = "" if sys.platform in ("win32", "msys", "cygwin"): env["PATH"] = env["PATH"] + ";" + self.xrePath bin_suffix = ".exe" else: if "LD_LIBRARY_PATH" not in env or env["LD_LIBRARY_PATH"] is None: env["LD_LIBRARY_PATH"] = self.xrePath else: env["LD_LIBRARY_PATH"] = ":".join( [self.xrePath, env["LD_LIBRARY_PATH"]]) args = [ "-g", self.xrePath, "-f", os.path.join(self.httpdPath, "httpd.js"), "-e", "const _PROFILE_PATH = '%(profile)s';const _SERVER_PORT = " "'%(port)s'; const _SERVER_ADDR ='%(server)s';" % { "profile": self.profileDir.replace("\\", "\\\\"), "port": self.httpPort, "server": self.webServer, }, "-f", os.path.join(self.scriptDir, "server.js"), ] xpcshell = os.path.join(self.utilityPath, "xpcshell" + bin_suffix) if not os.access(xpcshell, os.F_OK): raise Exception("xpcshell not found at %s" % xpcshell) if RemoteAutomation.elf_arm(xpcshell): raise Exception("xpcshell at %s is an ARM binary; please use " "the --utility-path argument to specify the path " "to a desktop version." % xpcshell) self._process = subprocess.Popen([xpcshell] + args, env=env) pid = self._process.pid if pid < 0: self.log.error( "TEST-UNEXPECTED-FAIL | remotereftests.py | Error starting server." ) return 2 self.log.info("INFO | remotereftests.py | Server pid: %d" % pid)
def main(args): automation = RemoteAutomation(None) parser = RemoteOptions(automation) options, args = parser.parse_args() if options.dm_trans == "sut" and options.deviceIP == None: print "Error: If --dm_trans = sut, you must provide a device IP to connect to via the --deviceIP option" return 1 try: if options.dm_trans == "adb": if options.deviceIP: dm = droid.DroidADB(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot) elif options.deviceSerial: dm = droid.DroidADB(None, None, deviceSerial=options.deviceSerial, deviceRoot=options.remoteTestRoot) else: dm = droid.DroidADB(None, None, deviceRoot=options.remoteTestRoot) else: dm = droid.DroidSUT(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot) except devicemanager.DMError: print "Automation Error: exception while initializing devicemanager. Most likely the device is not in a testable state." return 1 automation.setDeviceManager(dm) if options.remoteProductName != None: automation.setProduct(options.remoteProductName) # Set up the defaults and ensure options are set options = parser.verifyRemoteOptions(options) if options == None: print "ERROR: Invalid options specified, use --help for a list of valid options" return 1 if not options.ignoreWindowSize: parts = dm.getInfo("screen")["screen"][0].split() width = int(parts[0].split(":")[1]) height = int(parts[1].split(":")[1]) if width < 1050 or height < 1050: print "ERROR: Invalid screen resolution %sx%s, please adjust to 1366x1050 or higher" % (width, height) return 1 automation.setAppName(options.app) automation.setRemoteProfile(options.remoteProfile) automation.setRemoteLog(options.remoteLogFile) reftest = RemoteReftest(automation, dm, options, SCRIPT_DIRECTORY) options = parser.verifyCommonOptions(options, reftest) if mozinfo.info["debug"]: print "changing timeout for remote debug reftests from %s to 600 seconds" % options.timeout options.timeout = 600 # Hack in a symbolic link for jsreftest os.system("ln -s ../jsreftest " + str(os.path.join(SCRIPT_DIRECTORY, "jsreftest"))) # Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot manifest = args[0] if os.path.exists(os.path.join(SCRIPT_DIRECTORY, args[0])): manifest = "http://" + str(options.remoteWebServer) + ":" + str(options.httpPort) + "/" + args[0] elif os.path.exists(args[0]): manifestPath = os.path.abspath(args[0]).split(SCRIPT_DIRECTORY)[1].strip("/") manifest = "http://" + str(options.remoteWebServer) + ":" + str(options.httpPort) + "/" + manifestPath else: print "ERROR: Could not find test manifest '%s'" % manifest return 1 # Start the webserver retVal = reftest.startWebServer(options) if retVal: return retVal procName = options.app.split("/")[-1] if dm.processExist(procName): dm.killProcess(procName) reftest.printDeviceInfo() # an example manifest name to use on the cli # manifest = "http://" + options.remoteWebServer + "/reftests/layout/reftests/reftest-sanity/reftest.list" retVal = 0 try: cmdlineArgs = ["-reftest", manifest] if options.bootstrap: cmdlineArgs = [] dm.recordLogcat() retVal = reftest.runTests(manifest, options, cmdlineArgs) except: print "Automation Error: Exception caught while running tests" traceback.print_exc() retVal = 1 reftest.stopWebServer(options) reftest.printDeviceInfo(printLogcat=True) return retVal
def main(): auto = RemoteAutomation(None, "fennec") parser = RemoteOptions(auto) options, args = parser.parse_args() if (options.dm_trans == "adb"): if (options.deviceIP): dm = droid.DroidADB(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot) else: dm = droid.DroidADB(deviceRoot=options.remoteTestRoot) else: dm = droid.DroidSUT(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot) auto.setDeviceManager(dm) options = parser.verifyRemoteOptions(options, auto) if (options == None): print "ERROR: Invalid options specified, use --help for a list of valid options" sys.exit(1) productPieces = options.remoteProductName.split('.') if (productPieces != None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) mochitest = MochiRemote(auto, dm, options) options = parser.verifyOptions(options, mochitest) if (options == None): sys.exit(1) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent); auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) mochitest.printDeviceInfo() procName = options.app.split('/')[-1] if (dm.processExist(procName)): dm.killProcess(procName) if options.robocopIni != "": # sut may wait up to 300 s for a robocop am process before returning dm.default_timeout = 320 mp = manifestparser.TestManifest(strict=False) # TODO: pull this in dynamically mp.read(options.robocopIni) robocop_tests = mp.active_tests(exists=False) tests = [] my_tests = tests for test in robocop_tests: tests.append(test['name']) if options.totalChunks: tests_per_chunk = math.ceil(len(tests) / (options.totalChunks * 1.0)) start = int(round((options.thisChunk-1) * tests_per_chunk)) end = int(round(options.thisChunk * tests_per_chunk)) if end > len(tests): end = len(tests) my_tests = tests[start:end] print "Running tests %d-%d/%d" % ((start+1), end, len(tests)) deviceRoot = dm.getDeviceRoot() dm.removeFile(os.path.join(deviceRoot, "fennec_ids.txt")) fennec_ids = os.path.abspath("fennec_ids.txt") if not os.path.exists(fennec_ids) and options.robocopIds: fennec_ids = options.robocopIds dm.pushFile(fennec_ids, os.path.join(deviceRoot, "fennec_ids.txt")) options.extraPrefs.append('robocop.logfile="%s/robocop.log"' % deviceRoot) options.extraPrefs.append('browser.search.suggest.enabled=true') options.extraPrefs.append('browser.search.suggest.prompted=true') options.extraPrefs.append('layout.css.devPixelsPerPx="1.0"') options.extraPrefs.append('browser.chrome.dynamictoolbar=false') if (options.dm_trans == 'adb' and options.robocopApk): dm._checkCmd(["install", "-r", options.robocopApk]) retVal = None for test in robocop_tests: if options.testPath and options.testPath != test['name']: continue if not test['name'] in my_tests: continue # When running in a loop, we need to create a fresh profile for each cycle if mochitest.localProfile: options.profilePath = mochitest.localProfile os.system("rm -Rf %s" % options.profilePath) options.profilePath = tempfile.mkdtemp() mochitest.localProfile = options.profilePath options.app = "am" options.browserArgs = ["instrument", "-w", "-e", "deviceroot", deviceRoot, "-e", "class"] options.browserArgs.append("%s.tests.%s" % (options.remoteappname, test['name'])) options.browserArgs.append("org.mozilla.roboexample.test/%s.FennecInstrumentationTestRunner" % options.remoteappname) # If the test is for checking the import from bookmarks then make sure there is data to import if test['name'] == "testImportFromAndroid": # Get the OS so we can run the insert in the apropriate database and following the correct table schema osInfo = dm.getInfo("os") devOS = " ".join(osInfo['os']) if ("pandaboard" in devOS): delete = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"] else: delete = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"] if (options.dm_trans == "sut"): dm._runCmds([{"cmd": " ".join(delete)}]) # Insert the bookmarks print "Insert bookmarks in the default android browser database" for i in range(20): if ("pandaboard" in devOS): cmd = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser2.db 'insert or replace into bookmarks(_id,title,url,folder,parent,position) values (" + str(30 + i) + ",\"Bookmark"+ str(i) + "\",\"http://www.bookmark" + str(i) + ".com\",0,1," + str(100 + i) + ");'"] else: cmd = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser.db 'insert into bookmarks(title,url,bookmark) values (\"Bookmark"+ str(i) + "\",\"http://www.bookmark" + str(i) + ".com\",1);'"] if (options.dm_trans == "sut"): dm._runCmds([{"cmd": " ".join(cmd)}]) try: dm.removeDir("/mnt/sdcard/Robotium-Screenshots") dm.recordLogcat() result = mochitest.runTests(options) if result != 0: print "ERROR: runTests() exited with code %s" % result log_result = mochitest.addLogData() if result != 0 or log_result != 0: mochitest.printDeviceInfo(printLogcat=True) mochitest.printScreenshot() # Ensure earlier failures aren't overwritten by success on this run if retVal is None or retVal == 0: retVal = result except: print "Automation Error: Exception caught while running tests" traceback.print_exc() mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) try: mochitest.cleanup(None, options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 break finally: # Clean-up added bookmarks if test['name'] == "testImportFromAndroid": if ("pandaboard" in devOS): cmd_del = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"] else: cmd_del = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"] if (options.dm_trans == "sut"): dm._runCmds([{"cmd": " ".join(cmd_del)}]) if retVal is None: print "No tests run. Did you pass an invalid TEST_PATH?" retVal = 1 else: # if we didn't have some kind of error running the tests, make # sure the tests actually passed print "INFO | runtests.py | Test summary: start." overallResult = mochitest.printLog() print "INFO | runtests.py | Test summary: end." if retVal == 0: retVal = overallResult else: try: dm.recordLogcat() retVal = mochitest.runTests(options) except: print "Automation Error: Exception caught while running tests" traceback.print_exc() mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) try: mochitest.cleanup(None, options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 mochitest.printDeviceInfo(printLogcat=True) sys.exit(retVal)
def main(): scriptdir = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) auto = RemoteAutomation(None, "fennec") parser = RemoteOptions(auto, scriptdir) options, args = parser.parse_args() if (options.dm_trans == "adb"): if (options.deviceIP): dm = devicemanagerADB.DeviceManagerADB(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot) else: dm = devicemanagerADB.DeviceManagerADB(deviceRoot=options.remoteTestRoot) else: dm = devicemanagerSUT.DeviceManagerSUT(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot) auto.setDeviceManager(dm) options = parser.verifyRemoteOptions(options, auto) if (options == None): print "ERROR: Invalid options specified, use --help for a list of valid options" sys.exit(1) productPieces = options.remoteProductName.split('.') if (productPieces != None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) mochitest = MochiRemote(auto, dm, options) options = parser.verifyOptions(options, mochitest) if (options == None): sys.exit(1) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent); auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) print dm.getInfo() procName = options.app.split('/')[-1] if (dm.processExist(procName)): dm.killProcess(procName) if options.robocop != "": mp = manifestparser.TestManifest(strict=False) # TODO: pull this in dynamically mp.read(options.robocop) robocop_tests = mp.active_tests(exists=False) tests = [] my_tests = tests for test in robocop_tests: tests.append(test['name']) if options.totalChunks: tests_per_chunk = math.ceil(len(tests) / (options.totalChunks * 1.0)) start = int(round((options.thisChunk-1) * tests_per_chunk)) end = int(round(options.thisChunk * tests_per_chunk)) if end > len(tests): end = len(tests) my_tests = tests[start:end] print "Running tests %d-%d/%d" % ((start+1), end, len(tests)) deviceRoot = dm.getDeviceRoot() dm.removeFile(os.path.join(deviceRoot, "fennec_ids.txt")) fennec_ids = os.path.abspath("fennec_ids.txt") if not os.path.exists(fennec_ids) and options.robocopIds: fennec_ids = options.robocopIds dm.pushFile(fennec_ids, os.path.join(deviceRoot, "fennec_ids.txt")) options.extraPrefs.append('robocop.logfile="%s/robocop.log"' % deviceRoot) options.extraPrefs.append('browser.search.suggest.enabled=true') options.extraPrefs.append('browser.search.suggest.prompted=true') options.extraPrefs.append('browser.viewport.scaleRatio=100') options.extraPrefs.append('browser.chrome.dynamictoolbar=false') if (options.dm_trans == 'adb' and options.robocopPath): dm._checkCmd(["install", "-r", os.path.join(options.robocopPath, "robocop.apk")]) retVal = None for test in robocop_tests: if options.testPath and options.testPath != test['name']: continue if not test['name'] in my_tests: continue options.app = "am" options.browserArgs = ["instrument", "-w", "-e", "deviceroot", deviceRoot, "-e", "class"] options.browserArgs.append("%s.tests.%s" % (options.remoteappname, test['name'])) options.browserArgs.append("org.mozilla.roboexample.test/%s.FennecInstrumentationTestRunner" % options.remoteappname) try: dm.removeDir("/mnt/sdcard/Robotium-Screenshots") dm.recordLogcat() result = mochitest.runTests(options) if result != 0: print "ERROR: runTests() exited with code %s" % result log_result = mochitest.addLogData() if result != 0 or log_result != 0: mochitest.printDeviceInfo() mochitest.printScreenshot() # Ensure earlier failures aren't overwritten by success on this run if retVal is None or retVal == 0: retVal = result except: print "Automation Error: Exception caught while running tests" traceback.print_exc() mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) try: mochitest.cleanup(None, options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 break if retVal is None: print "No tests run. Did you pass an invalid TEST_PATH?" retVal = 1 else: # if we didn't have some kind of error running the tests, make # sure the tests actually passed print "INFO | runtests.py | Test summary: start." overallResult = mochitest.printLog() print "INFO | runtests.py | Test summary: end." if retVal == 0: retVal = overallResult else: try: dm.recordLogcat() retVal = mochitest.runTests(options) except: print "Automation Error: Exception caught while running tests" traceback.print_exc() mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) try: mochitest.cleanup(None, options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 mochitest.printDeviceInfo() sys.exit(retVal)
class RemoteReftest(RefTest): use_marionette = False resolver_cls = RemoteReftestResolver def __init__(self, options, scriptDir): RefTest.__init__(self, options.suite) self.run_by_manifest = False self.scriptDir = scriptDir self.localLogName = options.localLogName verbose = False if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug': verbose = True print "set verbose!" self.device = ADBDevice(adb=options.adb_path or 'adb', device=options.deviceSerial, test_root=options.remoteTestRoot, verbose=verbose) if options.remoteTestRoot is None: options.remoteTestRoot = posixpath.join(self.device.test_root, "reftest") options.remoteProfile = posixpath.join(options.remoteTestRoot, "profile") options.remoteLogFile = posixpath.join(options.remoteTestRoot, "reftest.log") options.logFile = options.remoteLogFile self.remoteProfile = options.remoteProfile self.remoteTestRoot = options.remoteTestRoot if not options.ignoreWindowSize: parts = self.device.get_info('screen')['screen'][0].split() width = int(parts[0].split(':')[1]) height = int(parts[1].split(':')[1]) if (width < 1366 or height < 1050): self.error("ERROR: Invalid screen resolution %sx%s, " "please adjust to 1366x1050 or higher" % (width, height)) self._populate_logger(options) self.outputHandler = OutputHandler(self.log, options.utilityPath, options.symbolsPath) # RemoteAutomation.py's 'messageLogger' is also used by mochitest. Mimic a mochitest # MessageLogger object to re-use this code path. self.outputHandler.write = self.outputHandler.__call__ args = {'messageLogger': self.outputHandler} self.automation = RemoteAutomation(self.device, appName=options.app, remoteProfile=self.remoteProfile, remoteLog=options.remoteLogFile, processArgs=args) self.environment = self.automation.environment if self.automation.IS_DEBUG_BUILD: self.SERVER_STARTUP_TIMEOUT = 180 else: self.SERVER_STARTUP_TIMEOUT = 90 self.remoteCache = os.path.join(options.remoteTestRoot, "cache/") # Check that Firefox is installed expected = options.app.split('/')[-1] if not self.device.is_app_installed(expected): raise Exception("%s is not installed on this device" % expected) self.device.clear_logcat() self.device.rm(self.remoteCache, force=True, recursive=True) procName = options.app.split('/')[-1] self.device.stop_application(procName) if self.device.process_exist(procName): self.log.error("unable to kill %s before starting tests!" % procName) def findPath(self, paths, filename=None): for path in paths: p = path if filename: p = os.path.join(p, filename) if os.path.exists(self.getFullPath(p)): return path return None def startWebServer(self, options): """ Create the webserver on the host and start it up """ remoteXrePath = options.xrePath remoteUtilityPath = options.utilityPath localAutomation = Automation() localAutomation.IS_WIN32 = False localAutomation.IS_LINUX = False localAutomation.IS_MAC = False localAutomation.UNIXISH = False hostos = sys.platform if (hostos == 'mac' or hostos == 'darwin'): localAutomation.IS_MAC = True elif (hostos == 'linux' or hostos == 'linux2'): localAutomation.IS_LINUX = True localAutomation.UNIXISH = True elif (hostos == 'win32' or hostos == 'win64'): localAutomation.BIN_SUFFIX = ".exe" localAutomation.IS_WIN32 = True paths = [options.xrePath, localAutomation.DIST_BIN] options.xrePath = self.findPath(paths) if options.xrePath is None: print( "ERROR: unable to find xulrunner path for %s, " "please specify with --xre-path" % (os.name)) return 1 paths.append("bin") paths.append(os.path.join("..", "bin")) xpcshell = "xpcshell" if (os.name == "nt"): xpcshell += ".exe" if (options.utilityPath): paths.insert(0, options.utilityPath) options.utilityPath = self.findPath(paths, xpcshell) if options.utilityPath is None: print( "ERROR: unable to find utility path for %s, " "please specify with --utility-path" % (os.name)) return 1 options.serverProfilePath = tempfile.mkdtemp() self.server = ReftestServer(localAutomation, options, self.scriptDir) retVal = self.server.start() if retVal: return retVal retVal = self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT) if retVal: return retVal options.xrePath = remoteXrePath options.utilityPath = remoteUtilityPath return 0 def stopWebServer(self, options): self.server.stop() def killNamedProc(self, pname, orphans=True): """ Kill processes matching the given command name """ self.log.info("Checking for %s processes..." % pname) for proc in psutil.process_iter(): try: if proc.name() == pname: procd = proc.as_dict( attrs=['pid', 'ppid', 'name', 'username']) if proc.ppid() == 1 or not orphans: self.log.info("killing %s" % procd) try: os.kill(proc.pid, getattr(signal, "SIGKILL", signal.SIGTERM)) except Exception as e: self.log.info("Failed to kill process %d: %s" % (proc.pid, str(e))) else: self.log.info("NOT killing %s (not an orphan?)" % procd) except Exception: # may not be able to access process info for all processes continue def createReftestProfile(self, options, **kwargs): profile = RefTest.createReftestProfile(self, options, server=options.remoteWebServer, port=options.httpPort, **kwargs) profileDir = profile.profile prefs = {} prefs["app.update.url.android"] = "" prefs["reftest.remote"] = True prefs[ "datareporting.policy.dataSubmissionPolicyBypassAcceptance"] = True # move necko cache to a location that can be cleaned up prefs["browser.cache.disk.parent_directory"] = self.remoteCache prefs["layout.css.devPixelsPerPx"] = "1.0" # Because Fennec is a little wacky (see bug 1156817) we need to load the # reftest pages at 1.0 zoom, rather than zooming to fit the CSS viewport. prefs["apz.allow_zooming"] = False # Set the extra prefs. profile.set_preferences(prefs) try: self.device.push(profileDir, options.remoteProfile) self.device.chmod(options.remoteProfile, recursive=True, root=True) except Exception: print "Automation Error: Failed to copy profiledir to device" raise return profile def printDeviceInfo(self, printLogcat=False): try: if printLogcat: logcat = self.device.get_logcat( filter_out_regexps=fennecLogcatFilters) for l in logcat: ul = l.decode('utf-8', errors='replace') sl = ul.encode('iso8859-1', errors='replace') print "%s\n" % sl print "Device info:" devinfo = self.device.get_info() for category in devinfo: if type(devinfo[category]) is list: print " %s:" % category for item in devinfo[category]: print " %s" % item else: print " %s: %s" % (category, devinfo[category]) print "Test root: %s" % self.device.test_root except ADBTimeoutError: raise except Exception as e: print "WARNING: Error getting device information: %s" % str(e) def environment(self, **kwargs): return self.automation.environment(**kwargs) def buildBrowserEnv(self, options, profileDir): browserEnv = RefTest.buildBrowserEnv(self, options, profileDir) # remove desktop environment not used on device if "XPCOM_MEM_BLOAT_LOG" in browserEnv: del browserEnv["XPCOM_MEM_BLOAT_LOG"] return browserEnv def runApp(self, options, cmdargs=None, timeout=None, debuggerInfo=None, symbolsPath=None, valgrindPath=None, valgrindArgs=None, valgrindSuppFiles=None, **profileArgs): if cmdargs is None: cmdargs = [] if self.use_marionette: cmdargs.append('-marionette') binary = options.app profile = self.createReftestProfile(options, **profileArgs) # browser environment env = self.buildBrowserEnv(options, profile.profile) self.log.info("Running with e10s: {}".format(options.e10s)) status, self.lastTestSeen = self.automation.runApp( None, env, binary, profile.profile, cmdargs, utilityPath=options.utilityPath, xrePath=options.xrePath, debuggerInfo=debuggerInfo, symbolsPath=symbolsPath, timeout=timeout, e10s=options.e10s) self.cleanup(profile.profile) return status def cleanup(self, profileDir): self.device.rm(self.remoteTestRoot, force=True, recursive=True) self.device.rm(self.remoteProfile, force=True, recursive=True) self.device.rm(self.remoteCache, force=True, recursive=True) RefTest.cleanup(self, profileDir)
def run_test_harness(options): message_logger = MessageLogger(logger=None) process_args = {'messageLogger': message_logger} auto = RemoteAutomation(None, "fennec", processArgs=process_args) if options is None: raise ValueError( "Invalid options specified, use --help for a list of valid options" ) dm = options.dm auto.setDeviceManager(dm) mochitest = MochiRemote(auto, dm, options) log = mochitest.log message_logger.logger = log mochitest.message_logger = message_logger productPieces = options.remoteProductName.split('.') if (productPieces is not None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) auto.setAppName(options.remoteappname) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent) auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) mochitest.printDeviceInfo() # Add Android version (SDK level) to mozinfo so that manifest entries # can be conditional on android_version. androidVersion = dm.shellCheckOutput(['getprop', 'ro.build.version.sdk']) log.info("Android sdk version '%s'; will use this to filter manifests" % str(androidVersion)) mozinfo.info['android_version'] = androidVersion deviceRoot = dm.deviceRoot if options.dmdPath: dmdLibrary = "libdmd.so" dmdPathOnDevice = os.path.join(deviceRoot, dmdLibrary) dm.removeFile(dmdPathOnDevice) dm.pushFile(os.path.join(options.dmdPath, dmdLibrary), dmdPathOnDevice) options.dmdPath = deviceRoot options.dumpOutputDirectory = deviceRoot procName = options.app.split('/')[-1] dm.killProcess(procName) if options.robocopIni != "": # turning buffering off as it's not used in robocop message_logger.buffering = False # sut may wait up to 300 s for a robocop am process before returning dm.default_timeout = 320 if isinstance(options.manifestFile, TestManifest): mp = options.manifestFile else: mp = TestManifest(strict=False) mp.read(options.robocopIni) filters = [] if options.totalChunks: filters.append( chunk_by_slice(options.thisChunk, options.totalChunks)) robocop_tests = mp.active_tests(exists=False, filters=filters, **mozinfo.info) options.extraPrefs.append('browser.search.suggest.enabled=true') options.extraPrefs.append('browser.search.suggest.prompted=true') options.extraPrefs.append('layout.css.devPixelsPerPx=1.0') options.extraPrefs.append('browser.chrome.dynamictoolbar=false') options.extraPrefs.append('browser.snippets.enabled=false') options.extraPrefs.append('browser.casting.enabled=true') if (options.dm_trans == 'adb' and options.robocopApk): dm._checkCmd(["install", "-r", options.robocopApk]) if not options.autorun: # Force a single loop iteration. The iteration will start Fennec and # the httpd server, but not actually run a test. options.testPath = robocop_tests[0]['name'] retVal = None # Filtering tests active_tests = [] for test in robocop_tests: if options.testPath and options.testPath != test['name']: continue if 'disabled' in test: log.info('TEST-INFO | skipping %s | %s' % (test['name'], test['disabled'])) continue active_tests.append(test) log.suite_start([t['name'] for t in active_tests]) for test in active_tests: # When running in a loop, we need to create a fresh profile for # each cycle if mochitest.localProfile: options.profilePath = mochitest.localProfile os.system("rm -Rf %s" % options.profilePath) options.profilePath = None mochitest.localProfile = options.profilePath options.app = "am" mochitest.nsprLogName = "nspr-%s.log" % test['name'] if options.autorun: # This launches a test (using "am instrument") and instructs # Fennec to /quit/ the browser (using Robocop:Quit) and to # /finish/ all opened activities. options.browserArgs = [ "instrument", "-w", "-e", "quit_and_finish", "1", "-e", "deviceroot", deviceRoot, "-e", "class" ] options.browserArgs.append("org.mozilla.gecko.tests.%s" % test['name'].split('.java')[0]) options.browserArgs.append( "org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner" ) else: # This does not launch a test at all. It launches an activity # that starts Fennec and then waits indefinitely, since cat # never returns. options.browserArgs = [ "start", "-n", "org.mozilla.roboexample.test/org.mozilla.gecko.LaunchFennecWithConfigurationActivity", "&&", "cat" ] dm.default_timeout = sys.maxint # Forever. mochitest.log.info("") mochitest.log.info( "Serving mochi.test Robocop root at http://%s:%s/tests/robocop/" % (options.remoteWebServer, options.httpPort)) mochitest.log.info("") # If the test is for checking the import from bookmarks then make # sure there is data to import if test['name'] == "testImportFromAndroid": # Get the OS so we can run the insert in the apropriate # database and following the correct table schema osInfo = dm.getInfo("os") devOS = " ".join(osInfo['os']) if ("pandaboard" in devOS): delete = [ 'execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'" ] else: delete = [ 'execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'" ] if (options.dm_trans == "sut"): dm._runCmds([{"cmd": " ".join(delete)}]) # Insert the bookmarks log.info( "Insert bookmarks in the default android browser database") for i in range(20): if ("pandaboard" in devOS): cmd = [ 'execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser2.db 'insert or replace into bookmarks(_id,title,url,folder,parent,position) values (" + str(30 + i) + ",\"Bookmark" + str(i) + "\",\"http://www.bookmark" + str(i) + ".com\",0,1," + str(100 + i) + ");'" ] else: cmd = [ 'execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser.db 'insert into bookmarks(title,url,bookmark) values (\"Bookmark" + str(i) + "\",\"http://www.bookmark" + str(i) + ".com\",1);'" ] if (options.dm_trans == "sut"): dm._runCmds([{"cmd": " ".join(cmd)}]) try: screenShotDir = "/mnt/sdcard/Robotium-Screenshots" dm.removeDir(screenShotDir) dm.recordLogcat() result = mochitest.runTests(options) if result != 0: log.error("runTests() exited with code %s" % result) log_result = mochitest.addLogData() if result != 0 or log_result != 0: mochitest.printDeviceInfo(printLogcat=True) mochitest.printScreenshots(screenShotDir) # Ensure earlier failures aren't overwritten by success on this # run if retVal is None or retVal == 0: retVal = result except: log.error( "Automation Error: Exception caught while running tests") traceback.print_exc() mochitest.stopServers() try: mochitest.cleanup(options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 break finally: # Clean-up added bookmarks if test['name'] == "testImportFromAndroid": if ("pandaboard" in devOS): cmd_del = [ 'execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'" ] else: cmd_del = [ 'execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'" ] if (options.dm_trans == "sut"): dm._runCmds([{"cmd": " ".join(cmd_del)}]) if retVal is None: log.warning("No tests run. Did you pass an invalid TEST_PATH?") retVal = 1 else: # if we didn't have some kind of error running the tests, make # sure the tests actually passed print "INFO | runtests.py | Test summary: start." overallResult = mochitest.printLog() print "INFO | runtests.py | Test summary: end." if retVal == 0: retVal = overallResult else: mochitest.nsprLogName = "nspr.log" try: dm.recordLogcat() retVal = mochitest.runTests(options) except: log.error("Automation Error: Exception caught while running tests") traceback.print_exc() mochitest.stopServers() try: mochitest.cleanup(options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 mochitest.printDeviceInfo(printLogcat=True) message_logger.finish() return retVal
def run_test_harness(parser, options): parser.validate(options) message_logger = MessageLogger(logger=None) process_args = {'messageLogger': message_logger} auto = RemoteAutomation(None, "fennec", processArgs=process_args) if options is None: raise ValueError( "Invalid options specified, use --help for a list of valid options" ) options.runByDir = False # roboextender is used by mochitest-chrome tests like test_java_addons.html, # but not by any plain mochitests if options.flavor != 'chrome': options.extensionsToExclude.append('*****@*****.**') dm = options.dm auto.setDeviceManager(dm) mochitest = MochiRemote(auto, dm, options) log = mochitest.log message_logger.logger = log mochitest.message_logger = message_logger # Check that Firefox is installed expected = options.app.split('/')[-1] installed = dm.shellCheckOutput(['pm', 'list', 'packages', expected]) if expected not in installed: log.error("%s is not installed on this device" % expected) return 1 productPieces = options.remoteProductName.split('.') if (productPieces is not None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) auto.setAppName(options.remoteappname) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent) auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) if options.log_mach is None: mochitest.printDeviceInfo() # Add Android version (SDK level) to mozinfo so that manifest entries # can be conditional on android_version. androidVersion = dm.shellCheckOutput(['getprop', 'ro.build.version.sdk']) log.info("Android sdk version '%s'; will use this to filter manifests" % str(androidVersion)) mozinfo.info['android_version'] = androidVersion deviceRoot = dm.deviceRoot if options.dmdPath: dmdLibrary = "libdmd.so" dmdPathOnDevice = os.path.join(deviceRoot, dmdLibrary) dm.removeFile(dmdPathOnDevice) dm.pushFile(os.path.join(options.dmdPath, dmdLibrary), dmdPathOnDevice) options.dmdPath = deviceRoot options.dumpOutputDirectory = deviceRoot procName = options.app.split('/')[-1] dm.killProcess(procName) mochitest.mozLogName = "moz.log" try: dm.recordLogcat() retVal = mochitest.runTests(options) except: log.error("Automation Error: Exception caught while running tests") traceback.print_exc() mochitest.stopServers() try: mochitest.cleanup(options) except mozdevice.DMError: # device error cleaning up... oh well! pass retVal = 1 if options.log_mach is None: mochitest.printDeviceInfo(printLogcat=True) message_logger.finish() return retVal
def run_test_harness(parser, options): parser.validate(options) message_logger = MessageLogger(logger=None) process_args = {"messageLogger": message_logger} auto = RemoteAutomation(None, "fennec", processArgs=process_args) if options is None: raise ValueError("Invalid options specified, use --help for a list of valid options") options.runByDir = False # roboextender is used by mochitest-chrome tests like test_java_addons.html, # but not by any plain mochitests if options.flavor != "chrome": options.extensionsToExclude.append("*****@*****.**") dm = options.dm auto.setDeviceManager(dm) mochitest = MochiRemote(auto, dm, options) log = mochitest.log message_logger.logger = log mochitest.message_logger = message_logger # Check that Firefox is installed expected = options.app.split("/")[-1] installed = dm.shellCheckOutput(["pm", "list", "packages", expected]) if expected not in installed: log.error("%s is not installed on this device" % expected) return 1 productPieces = options.remoteProductName.split(".") if productPieces is not None: auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) auto.setAppName(options.remoteappname) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent) auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) if options.log_mach is None: mochitest.printDeviceInfo() # Add Android version (SDK level) to mozinfo so that manifest entries # can be conditional on android_version. androidVersion = dm.shellCheckOutput(["getprop", "ro.build.version.sdk"]) log.info("Android sdk version '%s'; will use this to filter manifests" % str(androidVersion)) mozinfo.info["android_version"] = androidVersion deviceRoot = dm.deviceRoot if options.dmdPath: dmdLibrary = "libdmd.so" dmdPathOnDevice = os.path.join(deviceRoot, dmdLibrary) dm.removeFile(dmdPathOnDevice) dm.pushFile(os.path.join(options.dmdPath, dmdLibrary), dmdPathOnDevice) options.dmdPath = deviceRoot options.dumpOutputDirectory = deviceRoot procName = options.app.split("/")[-1] dm.killProcess(procName) mochitest.mozLogName = "moz.log" try: dm.recordLogcat() retVal = mochitest.runTests(options) except: log.error("Automation Error: Exception caught while running tests") traceback.print_exc() mochitest.stopServers() try: mochitest.cleanup(options) except mozdevice.DMError: # device error cleaning up... oh well! pass retVal = 1 if options.log_mach is None: mochitest.printDeviceInfo(printLogcat=True) message_logger.finish() return retVal
def run_test_harness(options): message_logger = MessageLogger(logger=None) process_args = {'messageLogger': message_logger} auto = RemoteAutomation(None, "fennec", processArgs=process_args) if options is None: raise ValueError( "Invalid options specified, use --help for a list of valid options" ) options.runByDir = False dm = options.dm auto.setDeviceManager(dm) mochitest = MochiRemote(auto, dm, options) log = mochitest.log message_logger.logger = log mochitest.message_logger = message_logger productPieces = options.remoteProductName.split('.') if (productPieces is not None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) auto.setAppName(options.remoteappname) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent) auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) if options.log_mach is None: mochitest.printDeviceInfo() # Add Android version (SDK level) to mozinfo so that manifest entries # can be conditional on android_version. androidVersion = dm.shellCheckOutput(['getprop', 'ro.build.version.sdk']) log.info("Android sdk version '%s'; will use this to filter manifests" % str(androidVersion)) mozinfo.info['android_version'] = androidVersion deviceRoot = dm.deviceRoot if options.dmdPath: dmdLibrary = "libdmd.so" dmdPathOnDevice = os.path.join(deviceRoot, dmdLibrary) dm.removeFile(dmdPathOnDevice) dm.pushFile(os.path.join(options.dmdPath, dmdLibrary), dmdPathOnDevice) options.dmdPath = deviceRoot options.dumpOutputDirectory = deviceRoot procName = options.app.split('/')[-1] dm.killProcess(procName) mochitest.nsprLogName = "nspr.log" try: dm.recordLogcat() retVal = mochitest.runTests(options) except: log.error("Automation Error: Exception caught while running tests") traceback.print_exc() mochitest.stopServers() try: mochitest.cleanup(options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 if options.log_mach is None: mochitest.printDeviceInfo(printLogcat=True) message_logger.finish() return retVal
def main(): scriptdir = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) dm_none = devicemanagerADB.DeviceManagerADB() auto = RemoteAutomation(dm_none, "fennec") parser = RemoteOptions(auto, scriptdir) options, args = parser.parse_args() if (options.dm_trans == "adb" or options.robocop): if (options.deviceIP): dm = devicemanagerADB.DeviceManagerADB(options.deviceIP, options.devicePort) else: dm = dm_none else: dm = devicemanagerSUT.DeviceManagerSUT(options.deviceIP, options.devicePort) auto.setDeviceManager(dm) options = parser.verifyRemoteOptions(options, auto) if (options == None): print "ERROR: Invalid options specified, use --help for a list of valid options" sys.exit(1) productPieces = options.remoteProductName.split('.') if (productPieces != None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) mochitest = MochiRemote(auto, dm, options) options = parser.verifyOptions(options, mochitest) if (options == None): sys.exit(1) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent); auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) procName = options.app.split('/')[-1] if (dm.processExist(procName)): dm.killProcess(procName) if (options.robocop): mp = manifestparser.TestManifest(strict=False) # TODO: pull this in dynamically mp.read('robocop.ini') robocop_tests = mp.active_tests(exists=False) fHandle = open("robotium.config", "w") fHandle.write("profile=%s\n" % (mochitest.remoteProfile)) fHandle.write("logfile=%s\n" % (options.remoteLogFile)) fHandle.close() deviceRoot = dm.getDeviceRoot() # Note, we are pushing to /sdcard since we have this location hard coded in robocop dm.pushFile("robotium.config", "/sdcard/robotium.config") dm.pushFile(os.path.abspath(options.robocop + "/fennec_ids.txt"), "/sdcard/fennec_ids.txt") options.extraPrefs.append('robocop.logfile="%s/robocop.log"' % deviceRoot) manifest = mochitest.buildProfile(options) mochitest.startWebServer(options) if (options.dm_trans == 'adb'): dm.checkCmd(["install", "-r", os.path.join(options.robocop, "robocop.apk")]) for test in robocop_tests: cmd = ["shell", "am", "instrument", "-w", "-e", "class"] cmd.append("%s.tests.%s" % (options.app, test['name'])) cmd.append("org.mozilla.roboexample.test/android.test.InstrumentationTestRunner") retVal = dm.checkCmd(cmd) else: # SUTAgent needs to install robocop and not crash when we launch robocop. retVal = dm.launchProcess(["am", "instrument", "-w", "org.mozilla.roboexample.test/android.test.InstrumentationTestRunner"]) mochitest.stopWebServer(options) else: try: retVal = mochitest.runTests(options) except: print "TEST-UNEXPECTED-ERROR | | Exception caught while running tests." mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) sys.exit(1) sys.exit(retVal)
def main(args): automation = RemoteAutomation(None) parser = RemoteOptions(automation) options, args = parser.parse_args() if (options.deviceIP == None): print "Error: you must provide a device IP to connect to via the --device option" return 1 try: if (options.dm_trans == "adb"): if (options.deviceIP): dm = devicemanagerADB.DeviceManagerADB(options.deviceIP, options.devicePort) else: dm = devicemanagerADB.DeviceManagerADB(None, None) else: dm = devicemanagerSUT.DeviceManagerSUT(options.deviceIP, options.devicePort) except devicemanager.DMError: print "Error: exception while initializing devicemanager. Most likely the device is not in a testable state." return 1 automation.setDeviceManager(dm) if (options.remoteProductName != None): automation.setProduct(options.remoteProductName) # Set up the defaults and ensure options are set options = parser.verifyRemoteOptions(options) if (options == None): print "ERROR: Invalid options specified, use --help for a list of valid options" return 1 if not options.ignoreWindowSize: parts = dm.getInfo('screen')['screen'][0].split() width = int(parts[0].split(':')[1]) height = int(parts[1].split(':')[1]) if (width < 1050 or height < 1050): print "ERROR: Invalid screen resolution %sx%s, please adjust to 1366x1050 or higher" % (width, height) return 1 automation.setAppName(options.app) automation.setRemoteProfile(options.remoteProfile) automation.setRemoteLog(options.remoteLogFile) reftest = RemoteReftest(automation, dm, options, SCRIPT_DIRECTORY) # Hack in a symbolic link for jsreftest os.system("ln -s ../jsreftest " + str(os.path.join(SCRIPT_DIRECTORY, "jsreftest"))) # Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot manifest = args[0] if os.path.exists(os.path.join(SCRIPT_DIRECTORY, args[0])): manifest = "http://" + str(options.remoteWebServer) + ":" + str(options.httpPort) + "/" + args[0] elif os.path.exists(args[0]): manifestPath = os.path.abspath(args[0]).split(SCRIPT_DIRECTORY)[1].strip('/') manifest = "http://" + str(options.remoteWebServer) + ":" + str(options.httpPort) + "/" + manifestPath else: print "ERROR: Could not find test manifest '%s'" % manifest return 1 # Start the webserver retVal = reftest.startWebServer(options) if retVal: return retVal procName = options.app.split('/')[-1] if (dm.processExist(procName)): dm.killProcess(procName) print dm.getInfo() #an example manifest name to use on the cli # manifest = "http://" + options.remoteWebServer + "/reftests/layout/reftests/reftest-sanity/reftest.list" retVal = 0 try: cmdlineArgs = ["-reftest", manifest] if options.bootstrap: cmdlineArgs = [] dm.recordLogcat() reftest.runTests(manifest, options, cmdlineArgs) except: print "Automation Error: Exception caught while running tests" traceback.print_exc() retVal = 1 reftest.stopWebServer(options) try: logcat = dm.getLogcat(filterOutRegexps=fennecLogcatFilters) print ''.join(logcat) print dm.getInfo() except devicemanager.DMError: print "WARNING: Error getting device information at end of test" return retVal
def main(): dm_none = DeviceManager(None, None) automation = RemoteAutomation(dm_none) parser = RemoteOptions(automation) options, args = parser.parse_args() if (options.deviceIP == None): print "Error: you must provide a device IP to connect to via the --device option" sys.exit(1) dm = DeviceManager(options.deviceIP, options.devicePort) automation.setDeviceManager(dm) if (options.remoteProductName != None): automation.setProduct(options.remoteProductName) # Set up the defaults and ensure options are set options = parser.verifyRemoteOptions(options) if (options == None): print "ERROR: Invalid options specified, use --help for a list of valid options" sys.exit(1) parts = dm.getInfo('screen')['screen'][0].split() width = int(parts[0].split(':')[1]) height = int(parts[1].split(':')[1]) if (width < 1050 or height < 1050): print "ERROR: Invalid screen resolution %sx%s, please adjust to 1366x1050 or higher" % (width, height) sys.exit(1) automation.setAppName(options.app) automation.setRemoteProfile(options.remoteProfile) automation.setRemoteLog(options.remoteLogFile) reftest = RemoteReftest(automation, dm, options, SCRIPT_DIRECTORY) # Start the webserver reftest.startWebServer(options) # Hack in a symbolic link for jsreftest os.system("ln -s ../jsreftest " + str(os.path.join(SCRIPT_DIRECTORY, "jsreftest"))) # Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot manifest = args[0] if os.path.exists(os.path.join(SCRIPT_DIRECTORY, args[0])): manifest = "http://" + str(options.remoteWebServer) + ":" + str(options.httpPort) + "/" + args[0] elif os.path.exists(args[0]): manifestPath = os.path.abspath(args[0]).split(SCRIPT_DIRECTORY)[1].strip('/') manifest = "http://" + str(options.remoteWebServer) + ":" + str(options.httpPort) + "/" + manifestPath procName = options.app.split('/')[-1] if (dm.processExist(procName)): dm.killProcess(procName) #an example manifest name to use on the cli # manifest = "http://" + options.remoteWebServer + "/reftests/layout/reftests/reftest-sanity/reftest.list" reftest.runTests(manifest, options) reftest.stopWebServer(options)
def run_test_harness(parser, options): parser.validate(options) message_logger = MessageLogger(logger=None) process_args = {'messageLogger': message_logger} auto = RemoteAutomation(None, "fennec", processArgs=process_args) if options is None: raise ValueError("Invalid options specified, use --help for a list of valid options") options.runByManifest = False # roboextender is used by mochitest-chrome tests like test_java_addons.html, # but not by any plain mochitests if options.flavor != 'chrome': options.extensionsToExclude.append('*****@*****.**') dm = options.dm auto.setDeviceManager(dm) mochitest = MochiRemote(auto, dm, options) options.dm = None log = mochitest.log message_logger.logger = log mochitest.message_logger = message_logger # Check that Firefox is installed expected = options.app.split('/')[-1] installed = dm.shellCheckOutput(['pm', 'list', 'packages', expected]) if expected not in installed: log.error("%s is not installed on this device" % expected) return 1 productPieces = options.remoteProductName.split('.') if (productPieces is not None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) auto.setAppName(options.remoteappname) logParent = os.path.dirname(options.remoteLogFile) dm.removeDir(logParent) dm.mkDir(logParent) auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) if options.log_mach is None: mochitest.printDeviceInfo() # Add Android version (SDK level) to mozinfo so that manifest entries # can be conditional on android_version. androidVersion = dm.shellCheckOutput(['getprop', 'ro.build.version.sdk']) log.info( "Android sdk version '%s'; will use this to filter manifests" % str(androidVersion)) mozinfo.info['android_version'] = androidVersion deviceRoot = dm.deviceRoot options.dumpOutputDirectory = deviceRoot procName = options.app.split('/')[-1] dm.killProcess(procName) if dm.processExist(procName): log.warning("unable to kill %s before running tests!" % procName) mochitest.mozLogName = "moz.log" try: dm.recordLogcat() if options.verify: retVal = mochitest.verifyTests(options) else: retVal = mochitest.runTests(options) except Exception: log.error("Automation Error: Exception caught while running tests") traceback.print_exc() mochitest.stopServers() try: mochitest.cleanup(options) except mozdevice.DMError: # device error cleaning up... oh well! pass retVal = 1 if options.log_mach is None: mochitest.printDeviceInfo(printLogcat=True) message_logger.finish() return retVal
def main(): scriptdir = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) dm_none = devicemanagerADB.DeviceManagerADB() auto = RemoteAutomation(dm_none, "fennec") parser = RemoteOptions(auto, scriptdir) options, args = parser.parse_args() if options.dm_trans == "adb": if options.deviceIP: dm = devicemanagerADB.DeviceManagerADB(options.deviceIP, options.devicePort) else: dm = dm_none else: dm = devicemanagerSUT.DeviceManagerSUT(options.deviceIP, options.devicePort) auto.setDeviceManager(dm) options = parser.verifyRemoteOptions(options, auto) if options == None: print "ERROR: Invalid options specified, use --help for a list of valid options" sys.exit(1) productPieces = options.remoteProductName.split(".") if productPieces != None: auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) mochitest = MochiRemote(auto, dm, options) options = parser.verifyOptions(options, mochitest) if options == None: sys.exit(1) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent) auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) procName = options.app.split("/")[-1] if dm.processExist(procName): dm.killProcess(procName) try: retVal = mochitest.runTests(options) except: print "TEST-UNEXPECTED-ERROR | | Exception caught while running tests." mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) sys.exit(1) sys.exit(retVal)
def main(): scriptdir = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) dm_none = devicemanagerADB.DeviceManagerADB() auto = RemoteAutomation(dm_none, "fennec") parser = RemoteOptions(auto, scriptdir) options, args = parser.parse_args() if (options.dm_trans == "adb"): if (options.deviceIP): dm = devicemanagerADB.DeviceManagerADB(options.deviceIP, options.devicePort) else: dm = dm_none else: dm = devicemanagerSUT.DeviceManagerSUT(options.deviceIP, options.devicePort) auto.setDeviceManager(dm) options = parser.verifyRemoteOptions(options, auto) if (options == None): print "ERROR: Invalid options specified, use --help for a list of valid options" sys.exit(1) productPieces = options.remoteProductName.split('.') if (productPieces != None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) mochitest = MochiRemote(auto, dm, options) options = parser.verifyOptions(options, mochitest) if (options == None): sys.exit(1) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent) auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) procName = options.app.split('/')[-1] if (dm.processExist(procName)): dm.killProcess(procName) if options.robocop != "": mp = manifestparser.TestManifest(strict=False) # TODO: pull this in dynamically mp.read(options.robocop) robocop_tests = mp.active_tests(exists=False) fHandle = open("robotium.config", "w") fHandle.write("profile=%s\n" % (mochitest.remoteProfile)) fHandle.write("logfile=%s\n" % (options.remoteLogFile)) fHandle.close() deviceRoot = dm.getDeviceRoot() # Note, we are pushing to /sdcard since we have this location hard coded in robocop dm.removeFile("/sdcard/fennec_ids.txt") dm.removeFile("/sdcard/robotium.config") dm.pushFile("robotium.config", "/sdcard/robotium.config") fennec_ids = os.path.abspath("fennec_ids.txt") if not os.path.exists(fennec_ids) and options.robocopPath: fennec_ids = os.path.abspath( os.path.join(options.robocopPath, "fennec_ids.txt")) dm.pushFile(fennec_ids, "/sdcard/fennec_ids.txt") options.extraPrefs.append('robocop.logfile="%s/robocop.log"' % deviceRoot) if (options.dm_trans == 'adb' and options.robocopPath): dm.checkCmd([ "install", "-r", os.path.join(options.robocopPath, "robocop.apk") ]) appname = options.app for test in robocop_tests: if options.testPath and options.testPath != test['name']: continue options.app = "am" options.browserArgs = ["instrument", "-w", "-e", "class"] options.browserArgs.append("%s.tests.%s" % (appname, test['name'])) options.browserArgs.append( "org.mozilla.roboexample.test/android.test.InstrumentationTestRunner" ) try: retVal = mochitest.runTests(options) except: print "TEST-UNEXPECTED-ERROR | %s | Exception caught while running robocop tests." % sys.exc_info( )[1] mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) try: self.cleanup(None, options) except: pass sys.exit(1) else: try: retVal = mochitest.runTests(options) except: print "TEST-UNEXPECTED-ERROR | %s | Exception caught while running tests." % sys.exc_info( )[1] mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) try: self.cleanup(None, options) except: pass sys.exit(1) sys.exit(retVal)
def main(): scriptdir = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) dm_none = devicemanagerADB.DeviceManagerADB() auto = RemoteAutomation(dm_none, "fennec") parser = RemoteOptions(auto, scriptdir) options, args = parser.parse_args() if (options.dm_trans == "adb"): if (options.deviceIP): dm = devicemanagerADB.DeviceManagerADB(options.deviceIP, options.devicePort) else: dm = dm_auto else: dm = devicemanagerSUT.DeviceManagerSUT(options.deviceIP, options.devicePort) auto.setDeviceManager(dm) options = parser.verifyRemoteOptions(options, auto) if (options == None): print "ERROR: Invalid options specified, use --help for a list of valid options" sys.exit(1) productPieces = options.remoteProductName.split('.') if (productPieces != None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) mochitest = MochiRemote(auto, dm, options) options = parser.verifyOptions(options, mochitest) if (options == None): sys.exit(1) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent); auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) procName = options.app.split('/')[-1] if (dm.processExist(procName)): dm.killProcess(procName) sys.exit(mochitest.runTests(options))
class RobocopTestRunner(MochitestDesktop): """ A test harness for Robocop. Robocop tests are UI tests for Firefox for Android, based on the Robotium test framework. This harness leverages some functionality from mochitest, for convenience. """ # Some robocop tests run for >60 seconds without generating any output. NO_OUTPUT_TIMEOUT = 180 def __init__(self, options, message_logger): """ Simple one-time initialization. """ MochitestDesktop.__init__(self, options.flavor, vars(options)) verbose = False if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug': verbose = True self.device = ADBDevice(adb=options.adbPath or 'adb', device=options.deviceSerial, test_root=options.remoteTestRoot, verbose=verbose) # Check that Firefox is installed expected = options.app.split('/')[-1] if not self.device.is_app_installed(expected): raise Exception("%s is not installed on this device" % expected) options.logFile = "robocop.log" if options.remoteTestRoot is None: options.remoteTestRoot = self.device.test_root self.remoteProfile = posixpath.join(options.remoteTestRoot, "profile") self.remoteProfileCopy = posixpath.join(options.remoteTestRoot, "profile-copy") self.remoteModulesDir = posixpath.join(options.remoteTestRoot, "modules/") self.remoteConfigFile = posixpath.join(options.remoteTestRoot, "robotium.config") self.remoteLogFile = posixpath.join(options.remoteTestRoot, "logs", "robocop.log") self.options = options process_args = {'messageLogger': message_logger} self.auto = RemoteAutomation(self.device, options.remoteappname, self.remoteProfile, self.remoteLogFile, processArgs=process_args) self.environment = self.auto.environment self.remoteScreenshots = "/mnt/sdcard/Robotium-Screenshots" self.remoteMozLog = posixpath.join(options.remoteTestRoot, "mozlog") self.localLog = options.logFile self.localProfile = None self.certdbNew = True self.passed = 0 self.failed = 0 self.todo = 0 def startup(self): """ Second-stage initialization: One-time initialization which may require cleanup. """ # Despite our efforts to clean up servers started by this script, in practice # we still see infrequent cases where a process is orphaned and interferes # with future tests, typically because the old server is keeping the port in use. # Try to avoid those failures by checking for and killing servers before # trying to start new ones. self.killNamedProc('ssltunnel') self.killNamedProc('xpcshell') self.auto.deleteANRs() self.auto.deleteTombstones() procName = self.options.app.split('/')[-1] self.device.stop_application(procName) if self.device.process_exist(procName): self.log.warning("unable to kill %s before running tests!" % procName) self.device.rm(self.remoteScreenshots, force=True, recursive=True) self.device.rm(self.remoteMozLog, force=True, recursive=True) self.device.mkdir(self.remoteMozLog) logParent = posixpath.dirname(self.remoteLogFile) self.device.rm(logParent, force=True, recursive=True) self.device.mkdir(logParent) # Add Android version (SDK level) to mozinfo so that manifest entries # can be conditional on android_version. self.log.info( "Android sdk version '%s'; will use this to filter manifests" % str(self.device.version)) mozinfo.info['android_version'] = str(self.device.version) if self.options.robocopApk: self.device.install_app(self.options.robocopApk, replace=True) self.log.debug("Robocop APK %s installed" % self.options.robocopApk) # Display remote diagnostics; if running in mach, keep output terse. if self.options.log_mach is None: self.printDeviceInfo() self.setupLocalPaths() self.buildProfile() self.startServers(self.options, debuggerInfo=None) self.log.debug("Servers started") def cleanup(self): """ Cleanup at end of job run. """ self.log.debug("Cleaning up...") self.stopServers() self.device.stop_application(self.options.app.split('/')[-1]) uploadDir = os.environ.get('MOZ_UPLOAD_DIR', None) if uploadDir: self.log.debug( "Pulling any remote moz logs and screenshots to %s." % uploadDir) if self.device.is_dir(self.remoteMozLog): self.device.pull(self.remoteMozLog, uploadDir) if self.device.is_dir(self.remoteScreenshots): self.device.pull(self.remoteScreenshots, uploadDir) MochitestDesktop.cleanup(self, self.options) if self.localProfile: mozfile.remove(self.localProfile) self.device.rm(self.remoteProfile, force=True, recursive=True) self.device.rm(self.remoteProfileCopy, force=True, recursive=True) self.device.rm(self.remoteScreenshots, force=True, recursive=True) self.device.rm(self.remoteMozLog, force=True, recursive=True) self.device.rm(self.remoteConfigFile, force=True) self.device.rm(self.remoteLogFile, force=True) self.log.debug("Cleanup complete.") def findPath(self, paths, filename=None): for path in paths: p = path if filename: p = os.path.join(p, filename) if os.path.exists(self.getFullPath(p)): return path return None def makeLocalAutomation(self): localAutomation = Automation() localAutomation.IS_WIN32 = False localAutomation.IS_LINUX = False localAutomation.IS_MAC = False localAutomation.UNIXISH = False hostos = sys.platform if (hostos == 'mac' or hostos == 'darwin'): localAutomation.IS_MAC = True elif (hostos == 'linux' or hostos == 'linux2'): localAutomation.IS_LINUX = True localAutomation.UNIXISH = True elif (hostos == 'win32' or hostos == 'win64'): localAutomation.BIN_SUFFIX = ".exe" localAutomation.IS_WIN32 = True return localAutomation def setupLocalPaths(self): """ Setup xrePath and utilityPath and verify xpcshell. This is similar to switchToLocalPaths in runtestsremote.py. """ localAutomation = self.makeLocalAutomation() paths = [self.options.xrePath, localAutomation.DIST_BIN] self.options.xrePath = self.findPath(paths) if self.options.xrePath is None: self.log.error( "unable to find xulrunner path for %s, please specify with --xre-path" % os.name) sys.exit(1) self.log.debug("using xre path %s" % self.options.xrePath) xpcshell = "xpcshell" if (os.name == "nt"): xpcshell += ".exe" if self.options.utilityPath: paths = [self.options.utilityPath, self.options.xrePath] else: paths = [self.options.xrePath] self.options.utilityPath = self.findPath(paths, xpcshell) if self.options.utilityPath is None: self.log.error( "unable to find utility path for %s, please specify with --utility-path" % os.name) sys.exit(1) self.log.debug("using utility path %s" % self.options.utilityPath) xpcshell_path = os.path.join(self.options.utilityPath, xpcshell) if localAutomation.elf_arm(xpcshell_path): self.log.error('xpcshell at %s is an ARM binary; please use ' 'the --utility-path argument to specify the path ' 'to a desktop version.' % xpcshell_path) sys.exit(1) self.log.debug("xpcshell found at %s" % xpcshell_path) def buildProfile(self): """ Build a profile locally, keep it locally for use by servers and push a copy to the remote profile-copy directory. This is similar to buildProfile in runtestsremote.py. """ self.options.extraPrefs.append('browser.search.suggest.enabled=true') self.options.extraPrefs.append('browser.search.suggest.prompted=true') self.options.extraPrefs.append('layout.css.devPixelsPerPx=1.0') self.options.extraPrefs.append('browser.chrome.dynamictoolbar=false') self.options.extraPrefs.append('browser.snippets.enabled=false') self.options.extraPrefs.append('extensions.autoupdate.enabled=false') # Override the telemetry init delay for integration testing. self.options.extraPrefs.append('toolkit.telemetry.initDelay=1') self.options.extensionsToExclude.extend([ '*****@*****.**', ]) self.extraPrefs = parse_preferences(self.options.extraPrefs) if self.options.testingModulesDir: try: self.device.push(self.options.testingModulesDir, self.remoteModulesDir) self.device.chmod(self.remoteModulesDir, recursive=True, root=True) except Exception: self.log.error( "Automation Error: Unable to copy test modules to device.") raise savedTestingModulesDir = self.options.testingModulesDir self.options.testingModulesDir = self.remoteModulesDir else: savedTestingModulesDir = None manifest = MochitestDesktop.buildProfile(self, self.options) if savedTestingModulesDir: self.options.testingModulesDir = savedTestingModulesDir self.localProfile = self.options.profilePath self.log.debug("Profile created at %s" % self.localProfile) # some files are not needed for robocop; save time by not pushing os.remove(os.path.join(self.localProfile, 'userChrome.css')) try: self.device.push(self.localProfile, self.remoteProfileCopy) except Exception: self.log.error( "Automation Error: Unable to copy profile to device.") raise return manifest def setupRemoteProfile(self): """ Remove any remote profile and re-create it. """ self.log.debug("Updating remote profile at %s" % self.remoteProfile) self.device.rm(self.remoteProfile, force=True, recursive=True) self.device.cp(self.remoteProfileCopy, self.remoteProfile, recursive=True) def parseLocalLog(self): """ Read and parse the local log file, noting any failures. """ with open(self.localLog) as currentLog: data = currentLog.readlines() os.unlink(self.localLog) start_found = False end_found = False fail_found = False for line in data: try: message = json.loads(line) if not isinstance(message, dict) or 'action' not in message: continue except ValueError: continue if message['action'] == 'test_end': end_found = True start_found = False break if start_found and not end_found: if 'status' in message: if 'expected' in message: self.failed += 1 elif message['status'] == 'PASS': self.passed += 1 elif message['status'] == 'FAIL': self.todo += 1 if message['action'] == 'test_start': start_found = True if 'expected' in message: fail_found = True result = 0 if fail_found: result = 1 if not end_found: self.log.info( "PROCESS-CRASH | Automation Error: Missing end of test marker (process crashed?)" ) result = 1 return result def logTestSummary(self): """ Print a summary of all tests run to stdout, for treeherder parsing (logging via self.log does not work here). """ print("0 INFO TEST-START | Shutdown") print("1 INFO Passed: %s" % (self.passed)) print("2 INFO Failed: %s" % (self.failed)) print("3 INFO Todo: %s" % (self.todo)) print("4 INFO SimpleTest FINISHED") if self.failed > 0: return 1 return 0 def printDeviceInfo(self, printLogcat=False): """ Log remote device information and logcat (if requested). This is similar to printDeviceInfo in runtestsremote.py """ try: if printLogcat: logcat = self.device.get_logcat( filter_out_regexps=fennecLogcatFilters) for l in logcat: self.log.info(l.decode('utf-8', 'replace')) self.log.info("Device info:") devinfo = self.device.get_info() for category in devinfo: if type(devinfo[category]) is list: self.log.info(" %s:" % category) for item in devinfo[category]: self.log.info(" %s" % item) else: self.log.info(" %s: %s" % (category, devinfo[category])) self.log.info("Test root: %s" % self.device.test_root) except ADBTimeoutError: raise except Exception as e: self.log.warning("Error getting device information: %s" % str(e)) def setupRobotiumConfig(self, browserEnv): """ Create robotium.config and push it to the device. """ fHandle = tempfile.NamedTemporaryFile(suffix='.config', prefix='robotium-', dir=os.getcwd(), delete=False) fHandle.write("profile=%s\n" % self.remoteProfile) fHandle.write("logfile=%s\n" % self.remoteLogFile) fHandle.write("host=http://mochi.test:8888/tests\n") fHandle.write("rawhost=http://%s:%s/tests\n" % (self.options.remoteWebServer, self.options.httpPort)) if browserEnv: envstr = "" delim = "" for key, value in browserEnv.items(): try: value.index(',') self.log.error( "setupRobotiumConfig: browserEnv - Found a ',' " "in our value, unable to process value. key=%s,value=%s" % (key, value)) self.log.error("browserEnv=%s" % browserEnv) except ValueError: envstr += "%s%s=%s" % (delim, key, value) delim = "," fHandle.write("envvars=%s\n" % envstr) fHandle.close() self.device.rm(self.remoteConfigFile, force=True) self.device.push(fHandle.name, self.remoteConfigFile) os.unlink(fHandle.name) def buildBrowserEnv(self): """ Return an environment dictionary suitable for remote use. This is similar to buildBrowserEnv in runtestsremote.py. """ browserEnv = self.environment(xrePath=None, debugger=None) # remove desktop environment not used on device if "XPCOM_MEM_BLOAT_LOG" in browserEnv: del browserEnv["XPCOM_MEM_BLOAT_LOG"] browserEnv["MOZ_LOG_FILE"] = os.path.join(self.remoteMozLog, self.mozLogName) try: browserEnv.update( dict( parse_key_value(self.options.environment, context='--setenv'))) except KeyValueParseError as e: self.log.error(str(e)) return None return browserEnv def runSingleTest(self, test): """ Run the specified test. """ self.log.debug("Running test %s" % test['name']) self.mozLogName = "moz-%s.log" % test['name'] browserEnv = self.buildBrowserEnv() self.setupRobotiumConfig(browserEnv) self.setupRemoteProfile() self.options.app = "am" timeout = None testName = test['name'].split('/')[-1].split('.java')[0] if self.options.enable_coverage: remoteCoverageFile = posixpath.join( self.options.remoteTestRoot, 'robocop-coverage-%s.ec' % testName) coverageFile = os.path.join(self.options.coverage_output_dir, 'robocop-coverage-%s.ec' % testName) if self.options.autorun: # This launches a test (using "am instrument") and instructs # Fennec to /quit/ the browser (using Robocop:Quit) and to # /finish/ all opened activities. browserArgs = [ "instrument", ] if self.options.enable_coverage: browserArgs += [ "-e", "coverage", "true", "-e", "coverageFile", remoteCoverageFile, ] browserArgs += [ "-e", "quit_and_finish", "1", "-e", "deviceroot", self.device.test_root, "-e", "class", "org.mozilla.gecko.tests.%s" % testName, "org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner", ] else: # This does not launch a test at all. It launches an activity # that starts Fennec and then waits indefinitely, since cat # never returns. browserArgs = [ "start", "-n", "org.mozilla.roboexample.test/org.mozilla." "gecko.LaunchFennecWithConfigurationActivity", "&&", "cat" ] timeout = sys.maxint # Forever. self.log.info("") self.log.info( "Serving mochi.test Robocop root at http://%s:%s/tests/robocop/" % (self.options.remoteWebServer, self.options.httpPort)) self.log.info("") result = -1 log_result = -1 try: self.device.clear_logcat() if not timeout: timeout = self.options.timeout if not timeout: timeout = self.NO_OUTPUT_TIMEOUT result, _ = self.auto.runApp(None, browserEnv, "am", self.localProfile, browserArgs, timeout=timeout, symbolsPath=self.options.symbolsPath) self.log.debug("runApp completes with status %d" % result) if result != 0: self.log.error("runApp() exited with code %s" % result) if self.device.is_file(self.remoteLogFile): self.device.pull(self.remoteLogFile, self.localLog) self.device.rm(self.remoteLogFile) log_result = self.parseLocalLog() if result != 0 or log_result != 0: # Display remote diagnostics; if running in mach, keep output # terse. if self.options.log_mach is None: self.printDeviceInfo(printLogcat=True) if self.options.enable_coverage: if self.device.is_file(remoteCoverageFile): self.device.pull(remoteCoverageFile, coverageFile) self.device.rm(remoteCoverageFile) else: self.log.warning( "Code coverage output not found on remote device: %s" % remoteCoverageFile) except Exception: self.log.error( "Automation Error: Exception caught while running tests") traceback.print_exc() result = 1 self.log.debug("Test %s completes with status %d (log status %d)" % (test['name'], int(result), int(log_result))) return result def runTests(self): self.startup() if isinstance(self.options.manifestFile, TestManifest): mp = self.options.manifestFile else: mp = TestManifest(strict=False) mp.read("robocop.ini") filters = [] if self.options.totalChunks: filters.append( chunk_by_slice(self.options.thisChunk, self.options.totalChunks)) robocop_tests = mp.active_tests(exists=False, filters=filters, **mozinfo.info) if not self.options.autorun: # Force a single loop iteration. The iteration will start Fennec and # the httpd server, but not actually run a test. self.options.test_paths = [robocop_tests[0]['name']] active_tests = [] for test in robocop_tests: if self.options.test_paths and test[ 'name'] not in self.options.test_paths: continue if 'disabled' in test: self.log.info('TEST-INFO | skipping %s | %s' % (test['name'], test['disabled'])) continue active_tests.append(test) tests_by_manifest = defaultdict(list) for test in active_tests: tests_by_manifest[test['manifest']].append(test['name']) self.log.suite_start(tests_by_manifest) worstTestResult = None for test in active_tests: result = self.runSingleTest(test) if worstTestResult is None or worstTestResult == 0: worstTestResult = result if worstTestResult is None: self.log.warning( "No tests run. Did you pass an invalid TEST_PATH?") worstTestResult = 1 else: print "INFO | runtests.py | Test summary: start." logResult = self.logTestSummary() print "INFO | runtests.py | Test summary: end." if worstTestResult == 0: worstTestResult = logResult return worstTestResult
def main(): auto = RemoteAutomation(None, "fennec") parser = RemoteOptions(auto) options, args = parser.parse_args() if (options.dm_trans == "adb"): if (options.deviceIP): dm = droid.DroidADB(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot) else: dm = droid.DroidADB(deviceRoot=options.remoteTestRoot) else: dm = droid.DroidSUT(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot) auto.setDeviceManager(dm) options = parser.verifyRemoteOptions(options, auto) if (options == None): log.error("Invalid options specified, use --help for a list of valid options") sys.exit(1) productPieces = options.remoteProductName.split('.') if (productPieces != None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) mochitest = MochiRemote(auto, dm, options) options = parser.verifyOptions(options, mochitest) if (options == None): sys.exit(1) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent); auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) mochitest.printDeviceInfo() procName = options.app.split('/')[-1] if (dm.processExist(procName)): dm.killProcess(procName) if options.robocopIni != "": # sut may wait up to 300 s for a robocop am process before returning dm.default_timeout = 320 mp = manifestparser.TestManifest(strict=False) # TODO: pull this in dynamically mp.read(options.robocopIni) robocop_tests = mp.active_tests(exists=False) tests = [] my_tests = tests for test in robocop_tests: tests.append(test['name']) if options.totalChunks: tests_per_chunk = math.ceil(len(tests) / (options.totalChunks * 1.0)) start = int(round((options.thisChunk-1) * tests_per_chunk)) end = int(round(options.thisChunk * tests_per_chunk)) if end > len(tests): end = len(tests) my_tests = tests[start:end] log.info("Running tests %d-%d/%d", start+1, end, len(tests)) deviceRoot = dm.getDeviceRoot() dm.removeFile(os.path.join(deviceRoot, "fennec_ids.txt")) fennec_ids = os.path.abspath("fennec_ids.txt") if not os.path.exists(fennec_ids) and options.robocopIds: fennec_ids = options.robocopIds dm.pushFile(fennec_ids, os.path.join(deviceRoot, "fennec_ids.txt")) options.extraPrefs.append('robocop.logfile="%s/robocop.log"' % deviceRoot) options.extraPrefs.append('browser.search.suggest.enabled=true') options.extraPrefs.append('browser.search.suggest.prompted=true') options.extraPrefs.append('layout.css.devPixelsPerPx="1.0"') options.extraPrefs.append('browser.chrome.dynamictoolbar=false') if (options.dm_trans == 'adb' and options.robocopApk): dm._checkCmd(["install", "-r", options.robocopApk]) retVal = None for test in robocop_tests: if options.testPath and options.testPath != test['name']: continue if not test['name'] in my_tests: continue # When running in a loop, we need to create a fresh profile for each cycle if mochitest.localProfile: options.profilePath = mochitest.localProfile os.system("rm -Rf %s" % options.profilePath) options.profilePath = tempfile.mkdtemp() mochitest.localProfile = options.profilePath options.app = "am" options.browserArgs = ["instrument", "-w", "-e", "deviceroot", deviceRoot, "-e", "class"] options.browserArgs.append("%s.tests.%s" % (options.remoteappname, test['name'])) options.browserArgs.append("org.mozilla.roboexample.test/%s.FennecInstrumentationTestRunner" % options.remoteappname) # If the test is for checking the import from bookmarks then make sure there is data to import if test['name'] == "testImportFromAndroid": # Get the OS so we can run the insert in the apropriate database and following the correct table schema osInfo = dm.getInfo("os") devOS = " ".join(osInfo['os']) # Bug 900664: stock browser db not available on x86 emulator if ("sdk_x86" in devOS): continue if ("pandaboard" in devOS): delete = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"] else: delete = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"] if (options.dm_trans == "sut"): dm._runCmds([{"cmd": " ".join(delete)}]) # Insert the bookmarks log.info("Insert bookmarks in the default android browser database") for i in range(20): if ("pandaboard" in devOS): cmd = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser2.db 'insert or replace into bookmarks(_id,title,url,folder,parent,position) values (" + str(30 + i) + ",\"Bookmark"+ str(i) + "\",\"http://www.bookmark" + str(i) + ".com\",0,1," + str(100 + i) + ");'"] else: cmd = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser.db 'insert into bookmarks(title,url,bookmark) values (\"Bookmark"+ str(i) + "\",\"http://www.bookmark" + str(i) + ".com\",1);'"] if (options.dm_trans == "sut"): dm._runCmds([{"cmd": " ".join(cmd)}]) try: dm.removeDir("/mnt/sdcard/Robotium-Screenshots") dm.recordLogcat() result = mochitest.runTests(options) if result != 0: log.error("runTests() exited with code %s", result) log_result = mochitest.addLogData() if result != 0 or log_result != 0: mochitest.printDeviceInfo(printLogcat=True) mochitest.printScreenshot() # Ensure earlier failures aren't overwritten by success on this run if retVal is None or retVal == 0: retVal = result except: log.error("Automation Error: Exception caught while running tests") traceback.print_exc() mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) try: mochitest.cleanup(None, options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 break finally: # Clean-up added bookmarks if test['name'] == "testImportFromAndroid": if ("pandaboard" in devOS): cmd_del = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"] else: cmd_del = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"] if (options.dm_trans == "sut"): dm._runCmds([{"cmd": " ".join(cmd_del)}]) if retVal is None: log.warn("No tests run. Did you pass an invalid TEST_PATH?") retVal = 1 else: # if we didn't have some kind of error running the tests, make # sure the tests actually passed print "INFO | runtests.py | Test summary: start." overallResult = mochitest.printLog() print "INFO | runtests.py | Test summary: end." if retVal == 0: retVal = overallResult else: try: dm.recordLogcat() retVal = mochitest.runTests(options) except: log.error("Automation Error: Exception caught while running tests") traceback.print_exc() mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) try: mochitest.cleanup(None, options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 mochitest.printDeviceInfo(printLogcat=True) sys.exit(retVal)
def main(): message_logger = MessageLogger(logger=None) process_args = {'messageLogger': message_logger} auto = RemoteAutomation(None, "fennec", processArgs=process_args) parser = RemoteOptions(auto) structured.commandline.add_logging_group(parser) options, args = parser.parse_args() if (options.dm_trans == "adb"): if (options.deviceIP): dm = droid.DroidADB(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot) else: dm = droid.DroidADB(deviceRoot=options.remoteTestRoot) else: dm = droid.DroidSUT(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot) auto.setDeviceManager(dm) options = parser.verifyRemoteOptions(options, auto) mochitest = MochiRemote(auto, dm, options) log = mochitest.log structured_logger = mochitest.structured_logger message_logger.logger = mochitest.structured_logger mochitest.message_logger = message_logger if (options == None): log.error("Invalid options specified, use --help for a list of valid options") sys.exit(1) productPieces = options.remoteProductName.split('.') if (productPieces != None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) auto.setAppName(options.remoteappname) options = parser.verifyOptions(options, mochitest) if (options == None): sys.exit(1) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent); auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) mochitest.printDeviceInfo() # Add Android version (SDK level) to mozinfo so that manifest entries # can be conditional on android_version. androidVersion = dm.shellCheckOutput(['getprop', 'ro.build.version.sdk']) log.info("Android sdk version '%s'; will use this to filter manifests" % str(androidVersion)) mozinfo.info['android_version'] = androidVersion deviceRoot = dm.deviceRoot if options.dmdPath: dmdLibrary = "libdmd.so" dmdPathOnDevice = os.path.join(deviceRoot, dmdLibrary) dm.removeFile(dmdPathOnDevice) dm.pushFile(os.path.join(options.dmdPath, dmdLibrary), dmdPathOnDevice) options.dmdPath = deviceRoot options.dumpOutputDirectory = deviceRoot procName = options.app.split('/')[-1] dm.killProcess(procName) if options.robocopIni != "": # turning buffering off as it's not used in robocop message_logger.buffering = False # sut may wait up to 300 s for a robocop am process before returning dm.default_timeout = 320 mp = manifestparser.TestManifest(strict=False) # TODO: pull this in dynamically mp.read(options.robocopIni) robocop_tests = mp.active_tests(exists=False, **mozinfo.info) tests = [] my_tests = tests for test in robocop_tests: tests.append(test['name']) if options.totalChunks: tests_per_chunk = math.ceil(len(tests) / (options.totalChunks * 1.0)) start = int(round((options.thisChunk-1) * tests_per_chunk)) end = int(round(options.thisChunk * tests_per_chunk)) if end > len(tests): end = len(tests) my_tests = tests[start:end] log.info("Running tests %d-%d/%d" % (start+1, end, len(tests))) dm.removeFile(os.path.join(deviceRoot, "fennec_ids.txt")) fennec_ids = os.path.abspath(os.path.join(SCRIPT_DIR, "fennec_ids.txt")) if not os.path.exists(fennec_ids) and options.robocopIds: fennec_ids = options.robocopIds dm.pushFile(fennec_ids, os.path.join(deviceRoot, "fennec_ids.txt")) options.extraPrefs.append('browser.search.suggest.enabled=true') options.extraPrefs.append('browser.search.suggest.prompted=true') options.extraPrefs.append('layout.css.devPixelsPerPx=1.0') options.extraPrefs.append('browser.chrome.dynamictoolbar=false') options.extraPrefs.append('browser.snippets.enabled=false') options.extraPrefs.append('browser.casting.enabled=true') if (options.dm_trans == 'adb' and options.robocopApk): dm._checkCmd(["install", "-r", options.robocopApk]) retVal = None # Filtering tests active_tests = [] for test in robocop_tests: if options.testPath and options.testPath != test['name']: continue if not test['name'] in my_tests: continue if 'disabled' in test: log.info('TEST-INFO | skipping %s | %s' % (test['name'], test['disabled'])) continue active_tests.append(test) structured_logger.suite_start([t['name'] for t in active_tests]) for test in active_tests: # When running in a loop, we need to create a fresh profile for each cycle if mochitest.localProfile: options.profilePath = mochitest.localProfile os.system("rm -Rf %s" % options.profilePath) options.profilePath = None mochitest.localProfile = options.profilePath options.app = "am" options.browserArgs = ["instrument", "-w", "-e", "deviceroot", deviceRoot, "-e", "class"] options.browserArgs.append("org.mozilla.gecko.tests.%s" % test['name']) options.browserArgs.append("org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner") # If the test is for checking the import from bookmarks then make sure there is data to import if test['name'] == "testImportFromAndroid": # Get the OS so we can run the insert in the apropriate database and following the correct table schema osInfo = dm.getInfo("os") devOS = " ".join(osInfo['os']) if ("pandaboard" in devOS): delete = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"] else: delete = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"] if (options.dm_trans == "sut"): dm._runCmds([{"cmd": " ".join(delete)}]) # Insert the bookmarks log.info("Insert bookmarks in the default android browser database") for i in range(20): if ("pandaboard" in devOS): cmd = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser2.db 'insert or replace into bookmarks(_id,title,url,folder,parent,position) values (" + str(30 + i) + ",\"Bookmark"+ str(i) + "\",\"http://www.bookmark" + str(i) + ".com\",0,1," + str(100 + i) + ");'"] else: cmd = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser.db 'insert into bookmarks(title,url,bookmark) values (\"Bookmark"+ str(i) + "\",\"http://www.bookmark" + str(i) + ".com\",1);'"] if (options.dm_trans == "sut"): dm._runCmds([{"cmd": " ".join(cmd)}]) try: screenShotDir = "/mnt/sdcard/Robotium-Screenshots" dm.removeDir(screenShotDir) dm.recordLogcat() result = mochitest.runTests(options) if result != 0: log.error("runTests() exited with code %s" % result) log_result = mochitest.addLogData() if result != 0 or log_result != 0: mochitest.printDeviceInfo(printLogcat=True) mochitest.printScreenshots(screenShotDir) # Ensure earlier failures aren't overwritten by success on this run if retVal is None or retVal == 0: retVal = result except: log.error("Automation Error: Exception caught while running tests") traceback.print_exc() mochitest.stopServers() try: mochitest.cleanup(options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 break finally: # Clean-up added bookmarks if test['name'] == "testImportFromAndroid": if ("pandaboard" in devOS): cmd_del = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"] else: cmd_del = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"] if (options.dm_trans == "sut"): dm._runCmds([{"cmd": " ".join(cmd_del)}]) if retVal is None: log.warning("No tests run. Did you pass an invalid TEST_PATH?") retVal = 1 else: # if we didn't have some kind of error running the tests, make # sure the tests actually passed print "INFO | runtests.py | Test summary: start." overallResult = mochitest.printLog() print "INFO | runtests.py | Test summary: end." if retVal == 0: retVal = overallResult else: try: dm.recordLogcat() retVal = mochitest.runTests(options) except: log.error("Automation Error: Exception caught while running tests") traceback.print_exc() mochitest.stopServers() try: mochitest.cleanup(options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 message_logger.finish() mochitest.printDeviceInfo(printLogcat=True) sys.exit(retVal)
def main(): scriptdir = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) auto = RemoteAutomation(None, "fennec") parser = RemoteOptions(auto, scriptdir) options, args = parser.parse_args() if (options.dm_trans == "adb"): if (options.deviceIP): dm = devicemanagerADB.DeviceManagerADB(options.deviceIP, options.devicePort) else: dm = devicemanagerADB.DeviceManagerADB() else: dm = devicemanagerSUT.DeviceManagerSUT(options.deviceIP, options.devicePort) auto.setDeviceManager(dm) options = parser.verifyRemoteOptions(options, auto) if (options == None): print "ERROR: Invalid options specified, use --help for a list of valid options" sys.exit(1) productPieces = options.remoteProductName.split('.') if (productPieces != None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) mochitest = MochiRemote(auto, dm, options) options = parser.verifyOptions(options, mochitest) if (options == None): sys.exit(1) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent); auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) print dm.getInfo() procName = options.app.split('/')[-1] if (dm.processExist(procName)): dm.killProcess(procName) if options.robocop != "": mp = manifestparser.TestManifest(strict=False) # TODO: pull this in dynamically mp.read(options.robocop) robocop_tests = mp.active_tests(exists=False) deviceRoot = dm.getDeviceRoot() dm.removeFile(os.path.join(deviceRoot, "fennec_ids.txt")) fennec_ids = os.path.abspath("fennec_ids.txt") if not os.path.exists(fennec_ids) and options.robocopIds: fennec_ids = options.robocopIds dm.pushFile(fennec_ids, os.path.join(deviceRoot, "fennec_ids.txt")) options.extraPrefs.append('robocop.logfile="%s/robocop.log"' % deviceRoot) options.extraPrefs.append('browser.search.suggest.enabled=true') options.extraPrefs.append('browser.search.suggest.prompted=true') if (options.dm_trans == 'adb' and options.robocopPath): dm._checkCmd(["install", "-r", os.path.join(options.robocopPath, "robocop.apk")]) appname = options.app retVal = None for test in robocop_tests: if options.testPath and options.testPath != test['name']: continue options.app = "am" options.browserArgs = ["instrument", "-w", "-e", "deviceroot", deviceRoot, "-e", "class"] options.browserArgs.append("%s.tests.%s" % (appname, test['name'])) options.browserArgs.append("org.mozilla.roboexample.test/%s.FennecInstrumentationTestRunner" % appname) try: dm.recordLogcat() retVal = mochitest.runTests(options) mochitest.addLogData() except: print "TEST-UNEXPECTED-FAIL | %s | Exception caught while running robocop tests." % sys.exc_info()[1] mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) try: mochitest.cleanup(None, options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 break if retVal is None: print "No tests run. Did you pass an invalid TEST_PATH?" retVal = 1 if retVal == 0: # if we didn't have some kind of error running the tests, make # sure the tests actually passed retVal = mochitest.printLog() else: try: dm.recordLogcat() retVal = mochitest.runTests(options) except: print "TEST-UNEXPECTED-FAIL | %s | Exception caught while running tests." % sys.exc_info()[1] mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) try: mochitest.cleanup(None, options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 try: logcat = dm.getLogcat() print ''.join(logcat[-500:-1]) print dm.getInfo() except devicemanager.DMError: print "WARNING: Error getting device information at end of test" sys.exit(retVal)
def main(): dm_none = devicemanagerADB.DeviceManagerADB(None, None) automation = RemoteAutomation(dm_none) parser = RemoteOptions(automation) options, args = parser.parse_args() if (options.deviceIP == None): print "Error: you must provide a device IP to connect to via the --device option" sys.exit(1) if (options.dm_trans == "adb"): if (options.deviceIP): dm = devicemanagerADB.DeviceManagerADB(options.deviceIP, options.devicePort) else: dm = dm_none else: dm = devicemanagerSUT.DeviceManagerSUT(options.deviceIP, options.devicePort) automation.setDeviceManager(dm) if (options.remoteProductName != None): automation.setProduct(options.remoteProductName) # Set up the defaults and ensure options are set options = parser.verifyRemoteOptions(options) if (options == None): print "ERROR: Invalid options specified, use --help for a list of valid options" sys.exit(1) if not options.ignoreWindowSize: parts = dm.getInfo('screen')['screen'][0].split() width = int(parts[0].split(':')[1]) height = int(parts[1].split(':')[1]) if (width < 1050 or height < 1050): print "ERROR: Invalid screen resolution %sx%s, please adjust to 1366x1050 or higher" % (width, height) sys.exit(1) automation.setAppName(options.app) automation.setRemoteProfile(options.remoteProfile) automation.setRemoteLog(options.remoteLogFile) reftest = RemoteReftest(automation, dm, options, SCRIPT_DIRECTORY) # Hack in a symbolic link for jsreftest os.system("ln -s ../jsreftest " + str(os.path.join(SCRIPT_DIRECTORY, "jsreftest"))) # Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot manifest = args[0] if os.path.exists(os.path.join(SCRIPT_DIRECTORY, args[0])): manifest = "http://" + str(options.remoteWebServer) + ":" + str(options.httpPort) + "/" + args[0] elif os.path.exists(args[0]): manifestPath = os.path.abspath(args[0]).split(SCRIPT_DIRECTORY)[1].strip('/') manifest = "http://" + str(options.remoteWebServer) + ":" + str(options.httpPort) + "/" + manifestPath else: print "ERROR: Could not find test manifest '%s'" % manifest sys.exit(1) # Start the webserver reftest.startWebServer(options) procName = options.app.split('/')[-1] if (dm.processExist(procName)): dm.killProcess(procName) #an example manifest name to use on the cli # manifest = "http://" + options.remoteWebServer + "/reftests/layout/reftests/reftest-sanity/reftest.list" try: cmdlineArgs = ["-reftest", manifest] if options.bootstrap: cmdlineArgs = [] reftest.runTests(manifest, options, cmdlineArgs) except: print "TEST-UNEXPECTED-FAIL | | exception while running reftests" reftest.stopWebServer(options) sys.exit(1) reftest.stopWebServer(options)
def main(): scriptdir = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) auto = RemoteAutomation(None, "fennec") parser = RemoteOptions(auto, scriptdir) options, args = parser.parse_args() if (options.dm_trans == "adb"): if (options.deviceIP): dm = devicemanagerADB.DeviceManagerADB(options.deviceIP, options.devicePort) else: dm = devicemanagerADB.DeviceManagerADB() else: dm = devicemanagerSUT.DeviceManagerSUT(options.deviceIP, options.devicePort) auto.setDeviceManager(dm) options = parser.verifyRemoteOptions(options, auto) if (options == None): print "ERROR: Invalid options specified, use --help for a list of valid options" sys.exit(1) productPieces = options.remoteProductName.split('.') if (productPieces != None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) mochitest = MochiRemote(auto, dm, options) options = parser.verifyOptions(options, mochitest) if (options == None): sys.exit(1) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent); auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) print dm.getInfo() procName = options.app.split('/')[-1] if (dm.processExist(procName)): dm.killProcess(procName) if options.robocop != "": mp = manifestparser.TestManifest(strict=False) # TODO: pull this in dynamically mp.read(options.robocop) robocop_tests = mp.active_tests(exists=False) fHandle = open("robotium.config", "w") fHandle.write("profile=%s\n" % (mochitest.remoteProfile)) fHandle.write("logfile=%s\n" % (options.remoteLogFile)) fHandle.write("host=http://mochi.test:8888/tests\n") fHandle.write("rawhost=http://%s:%s/tests\n" % (options.remoteWebServer, options.httpPort)) fHandle.close() deviceRoot = dm.getDeviceRoot() dm.removeFile(os.path.join(deviceRoot, "fennec_ids.txt")) dm.removeFile(os.path.join(deviceRoot, "robotium.config")) dm.pushFile("robotium.config", os.path.join(deviceRoot, "robotium.config")) fennec_ids = os.path.abspath("fennec_ids.txt") if not os.path.exists(fennec_ids) and options.robocopIds: fennec_ids = options.robocopIds dm.pushFile(fennec_ids, os.path.join(deviceRoot, "fennec_ids.txt")) options.extraPrefs.append('robocop.logfile="%s/robocop.log"' % deviceRoot) if (options.dm_trans == 'adb' and options.robocopPath): dm.checkCmd(["install", "-r", os.path.join(options.robocopPath, "robocop.apk")]) appname = options.app retVal = None logcat = [] for test in robocop_tests: if options.testPath and options.testPath != test['name']: continue options.app = "am" options.browserArgs = ["instrument", "-w", "-e", "deviceroot", deviceRoot, "-e", "class"] options.browserArgs.append("%s.tests.%s" % (appname, test['name'])) options.browserArgs.append("org.mozilla.roboexample.test/%s.FennecInstrumentationTestRunner" % appname) try: dm.recordLogcat() retVal = mochitest.runTests(options) logcat = dm.getLogcat() mochitest.addLogData() except: print "TEST-UNEXPECTED-FAIL | %s | Exception caught while running robocop tests." % sys.exc_info()[1] mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) try: self.cleanup(None, options) except: pass sys.exit(1) if retVal is None: print "No tests run. Did you pass an invalid TEST_PATH?" retVal = 1 retVal = mochitest.printLog() else: try: dm.recordLogcat() retVal = mochitest.runTests(options) logcat = dm.getLogcat() except: print "TEST-UNEXPECTED-FAIL | %s | Exception caught while running tests." % sys.exc_info()[1] mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) try: self.cleanup(None, options) except: pass sys.exit(1) print ''.join(logcat[-500:-1]) print dm.getInfo() sys.exit(retVal)
def main(): scriptdir = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) auto = RemoteAutomation(None, "fennec") parser = RemoteOptions(auto, scriptdir) options, args = parser.parse_args() if (options.dm_trans == "adb"): if (options.deviceIP): dm = devicemanagerADB.DeviceManagerADB(options.deviceIP, options.devicePort) else: dm = devicemanagerADB.DeviceManagerADB() else: dm = devicemanagerSUT.DeviceManagerSUT(options.deviceIP, options.devicePort) auto.setDeviceManager(dm) options = parser.verifyRemoteOptions(options, auto) if (options == None): print "ERROR: Invalid options specified, use --help for a list of valid options" sys.exit(1) productPieces = options.remoteProductName.split('.') if (productPieces != None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) mochitest = MochiRemote(auto, dm, options) options = parser.verifyOptions(options, mochitest) if (options == None): sys.exit(1) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent) auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) print dm.getInfo() procName = options.app.split('/')[-1] if (dm.processExist(procName)): dm.killProcess(procName) if options.robocop != "": mp = manifestparser.TestManifest(strict=False) # TODO: pull this in dynamically mp.read(options.robocop) robocop_tests = mp.active_tests(exists=False) fHandle = open("robotium.config", "w") fHandle.write("profile=%s\n" % (mochitest.remoteProfile)) fHandle.write("logfile=%s\n" % (options.remoteLogFile)) fHandle.write("host=http://mochi.test:8888/tests\n") fHandle.write("rawhost=http://%s:%s/tests\n" % (options.remoteWebServer, options.httpPort)) fHandle.close() deviceRoot = dm.getDeviceRoot() dm.removeFile(os.path.join(deviceRoot, "fennec_ids.txt")) dm.removeFile(os.path.join(deviceRoot, "robotium.config")) dm.pushFile("robotium.config", os.path.join(deviceRoot, "robotium.config")) fennec_ids = os.path.abspath("fennec_ids.txt") if not os.path.exists(fennec_ids) and options.robocopIds: fennec_ids = options.robocopIds dm.pushFile(fennec_ids, os.path.join(deviceRoot, "fennec_ids.txt")) options.extraPrefs.append('robocop.logfile="%s/robocop.log"' % deviceRoot) if (options.dm_trans == 'adb' and options.robocopPath): dm.checkCmd([ "install", "-r", os.path.join(options.robocopPath, "robocop.apk") ]) appname = options.app retVal = None logcat = [] for test in robocop_tests: if options.testPath and options.testPath != test['name']: continue options.app = "am" options.browserArgs = [ "instrument", "-w", "-e", "deviceroot", deviceRoot, "-e", "class" ] options.browserArgs.append("%s.tests.%s" % (appname, test['name'])) options.browserArgs.append( "org.mozilla.roboexample.test/%s.FennecInstrumentationTestRunner" % appname) try: dm.recordLogcat() retVal = mochitest.runTests(options) logcat = dm.getLogcat() mochitest.addLogData() except: print "TEST-UNEXPECTED-FAIL | %s | Exception caught while running robocop tests." % sys.exc_info( )[1] mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) try: self.cleanup(None, options) except: pass sys.exit(1) if retVal is None: print "No tests run. Did you pass an invalid TEST_PATH?" retVal = 1 retVal = mochitest.printLog() else: try: dm.recordLogcat() retVal = mochitest.runTests(options) logcat = dm.getLogcat() except: print "TEST-UNEXPECTED-FAIL | %s | Exception caught while running tests." % sys.exc_info( )[1] mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) try: self.cleanup(None, options) except: pass sys.exit(1) print ''.join(logcat[-500:-1]) print dm.getInfo() sys.exit(retVal)
def __init__(self, options, scriptDir): RefTest.__init__(self, options.suite) self.run_by_manifest = False self.scriptDir = scriptDir self.localLogName = options.localLogName verbose = False if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug': verbose = True print "set verbose!" self.device = ADBDevice(adb=options.adb_path or 'adb', device=options.deviceSerial, test_root=options.remoteTestRoot, verbose=verbose) if options.remoteTestRoot is None: options.remoteTestRoot = posixpath.join(self.device.test_root, "reftest") options.remoteProfile = posixpath.join(options.remoteTestRoot, "profile") options.remoteLogFile = posixpath.join(options.remoteTestRoot, "reftest.log") options.logFile = options.remoteLogFile self.remoteProfile = options.remoteProfile self.remoteTestRoot = options.remoteTestRoot if not options.ignoreWindowSize: parts = self.device.get_info('screen')['screen'][0].split() width = int(parts[0].split(':')[1]) height = int(parts[1].split(':')[1]) if (width < 1366 or height < 1050): self.error("ERROR: Invalid screen resolution %sx%s, " "please adjust to 1366x1050 or higher" % (width, height)) self._populate_logger(options) self.outputHandler = OutputHandler(self.log, options.utilityPath, options.symbolsPath) # RemoteAutomation.py's 'messageLogger' is also used by mochitest. Mimic a mochitest # MessageLogger object to re-use this code path. self.outputHandler.write = self.outputHandler.__call__ args = {'messageLogger': self.outputHandler} self.automation = RemoteAutomation(self.device, appName=options.app, remoteProfile=self.remoteProfile, remoteLog=options.remoteLogFile, processArgs=args) self.environment = self.automation.environment if self.automation.IS_DEBUG_BUILD: self.SERVER_STARTUP_TIMEOUT = 180 else: self.SERVER_STARTUP_TIMEOUT = 90 self.remoteCache = os.path.join(options.remoteTestRoot, "cache/") # Check that Firefox is installed expected = options.app.split('/')[-1] if not self.device.is_app_installed(expected): raise Exception("%s is not installed on this device" % expected) self.device.clear_logcat() self.device.rm(self.remoteCache, force=True, recursive=True) procName = options.app.split('/')[-1] self.device.stop_application(procName) if self.device.process_exist(procName): self.log.error("unable to kill %s before starting tests!" % procName)
def main(): scriptdir = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) dm_none = devicemanagerADB.DeviceManagerADB() auto = RemoteAutomation(dm_none, "fennec") parser = RemoteOptions(auto, scriptdir) options, args = parser.parse_args() if (options.dm_trans == "adb"): if (options.deviceIP): dm = devicemanagerADB.DeviceManagerADB(options.deviceIP, options.devicePort) else: dm = dm_auto else: dm = devicemanagerSUT.DeviceManagerSUT(options.deviceIP, options.devicePort) auto.setDeviceManager(dm) options = parser.verifyRemoteOptions(options, auto) if (options == None): print "ERROR: Invalid options specified, use --help for a list of valid options" sys.exit(1) productPieces = options.remoteProductName.split('.') if (productPieces != None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) mochitest = MochiRemote(auto, dm, options) options = parser.verifyOptions(options, mochitest) if (options == None): sys.exit(1) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent) auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) procName = options.app.split('/')[-1] if (dm.processExist(procName)): dm.killProcess(procName) sys.exit(mochitest.runTests(options))
def run_test_harness(options): message_logger = MessageLogger(logger=None) process_args = {'messageLogger': message_logger} auto = RemoteAutomation(None, "fennec", processArgs=process_args) if options is None: raise ValueError("Invalid options specified, use --help for a list of valid options") dm = options.dm auto.setDeviceManager(dm) mochitest = MochiRemote(auto, dm, options) log = mochitest.log message_logger.logger = log mochitest.message_logger = message_logger productPieces = options.remoteProductName.split('.') if (productPieces is not None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) auto.setAppName(options.remoteappname) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent) auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) mochitest.printDeviceInfo() # Add Android version (SDK level) to mozinfo so that manifest entries # can be conditional on android_version. androidVersion = dm.shellCheckOutput(['getprop', 'ro.build.version.sdk']) log.info( "Android sdk version '%s'; will use this to filter manifests" % str(androidVersion)) mozinfo.info['android_version'] = androidVersion deviceRoot = dm.deviceRoot if options.dmdPath: dmdLibrary = "libdmd.so" dmdPathOnDevice = os.path.join(deviceRoot, dmdLibrary) dm.removeFile(dmdPathOnDevice) dm.pushFile(os.path.join(options.dmdPath, dmdLibrary), dmdPathOnDevice) options.dmdPath = deviceRoot options.dumpOutputDirectory = deviceRoot procName = options.app.split('/')[-1] dm.killProcess(procName) if options.robocopIni != "": # turning buffering off as it's not used in robocop message_logger.buffering = False # sut may wait up to 300 s for a robocop am process before returning dm.default_timeout = 320 if isinstance(options.manifestFile, TestManifest): mp = options.manifestFile else: mp = TestManifest(strict=False) mp.read(options.robocopIni) filters = [] if options.totalChunks: filters.append( chunk_by_slice(options.thisChunk, options.totalChunks)) robocop_tests = mp.active_tests(exists=False, filters=filters, **mozinfo.info) options.extraPrefs.append('browser.search.suggest.enabled=true') options.extraPrefs.append('browser.search.suggest.prompted=true') options.extraPrefs.append('layout.css.devPixelsPerPx=1.0') options.extraPrefs.append('browser.chrome.dynamictoolbar=false') options.extraPrefs.append('browser.snippets.enabled=false') options.extraPrefs.append('browser.casting.enabled=true') options.extraPrefs.append('extensions.autoupdate.enabled=false') if (options.dm_trans == 'adb' and options.robocopApk): dm._checkCmd(["install", "-r", options.robocopApk]) if not options.autorun: # Force a single loop iteration. The iteration will start Fennec and # the httpd server, but not actually run a test. options.test_paths = [robocop_tests[0]['name']] retVal = None # Filtering tests active_tests = [] for test in robocop_tests: if options.test_paths and test['name'] not in options.test_paths: continue if 'disabled' in test: log.info( 'TEST-INFO | skipping %s | %s' % (test['name'], test['disabled'])) continue active_tests.append(test) log.suite_start([t['name'] for t in active_tests]) for test in active_tests: # When running in a loop, we need to create a fresh profile for # each cycle if mochitest.localProfile: options.profilePath = mochitest.localProfile os.system("rm -Rf %s" % options.profilePath) options.profilePath = None mochitest.localProfile = options.profilePath options.app = "am" mochitest.nsprLogName = "nspr-%s.log" % test['name'] if options.autorun: # This launches a test (using "am instrument") and instructs # Fennec to /quit/ the browser (using Robocop:Quit) and to # /finish/ all opened activities. options.browserArgs = [ "instrument", "-w", "-e", "quit_and_finish", "1", "-e", "deviceroot", deviceRoot, "-e", "class"] options.browserArgs.append( "org.mozilla.gecko.tests.%s" % test['name'].split('.java')[0]) options.browserArgs.append( "org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner") else: # This does not launch a test at all. It launches an activity # that starts Fennec and then waits indefinitely, since cat # never returns. options.browserArgs = ["start", "-n", "org.mozilla.roboexample.test/org.mozilla.gecko.LaunchFennecWithConfigurationActivity", "&&", "cat"] dm.default_timeout = sys.maxint # Forever. mochitest.log.info("") mochitest.log.info("Serving mochi.test Robocop root at http://%s:%s/tests/robocop/" % (options.remoteWebServer, options.httpPort)) mochitest.log.info("") # If the test is for checking the import from bookmarks then make # sure there is data to import if test['name'] == "testImportFromAndroid": # Get the OS so we can run the insert in the apropriate # database and following the correct table schema osInfo = dm.getInfo("os") devOS = " ".join(osInfo['os']) if ("pandaboard" in devOS): delete = [ 'execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"] else: delete = [ 'execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"] if (options.dm_trans == "sut"): dm._runCmds([{"cmd": " ".join(delete)}]) # Insert the bookmarks log.info( "Insert bookmarks in the default android browser database") for i in range(20): if ("pandaboard" in devOS): cmd = [ 'execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser2.db 'insert or replace into bookmarks(_id,title,url,folder,parent,position) values (" + str( 30 + i) + ",\"Bookmark" + str(i) + "\",\"http://www.bookmark" + str(i) + ".com\",0,1," + str( 100 + i) + ");'"] else: cmd = [ 'execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser.db 'insert into bookmarks(title,url,bookmark) values (\"Bookmark" + str(i) + "\",\"http://www.bookmark" + str(i) + ".com\",1);'"] if (options.dm_trans == "sut"): dm._runCmds([{"cmd": " ".join(cmd)}]) try: screenShotDir = "/mnt/sdcard/Robotium-Screenshots" dm.removeDir(screenShotDir) dm.recordLogcat() result = mochitest.runTests(options) if result != 0: log.error("runTests() exited with code %s" % result) log_result = mochitest.addLogData() if result != 0 or log_result != 0: mochitest.printDeviceInfo(printLogcat=True) mochitest.printScreenshots(screenShotDir) # Ensure earlier failures aren't overwritten by success on this # run if retVal is None or retVal == 0: retVal = result except: log.error( "Automation Error: Exception caught while running tests") traceback.print_exc() mochitest.stopServers() try: mochitest.cleanup(options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 break finally: # Clean-up added bookmarks if test['name'] == "testImportFromAndroid": if ("pandaboard" in devOS): cmd_del = [ 'execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"] else: cmd_del = [ 'execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"] if (options.dm_trans == "sut"): dm._runCmds([{"cmd": " ".join(cmd_del)}]) if retVal is None: log.warning("No tests run. Did you pass an invalid TEST_PATH?") retVal = 1 else: # if we didn't have some kind of error running the tests, make # sure the tests actually passed print "INFO | runtests.py | Test summary: start." overallResult = mochitest.printLog() print "INFO | runtests.py | Test summary: end." if retVal == 0: retVal = overallResult else: mochitest.nsprLogName = "nspr.log" try: dm.recordLogcat() retVal = mochitest.runTests(options) except: log.error("Automation Error: Exception caught while running tests") traceback.print_exc() mochitest.stopServers() try: mochitest.cleanup(options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 mochitest.printDeviceInfo(printLogcat=True) message_logger.finish() return retVal
class MochiRemote(MochitestDesktop): localProfile = None logMessages = [] def __init__(self, options): MochitestDesktop.__init__(self, options.flavor, vars(options)) verbose = False if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug': verbose = True if hasattr(options, 'log'): delattr(options, 'log') self.certdbNew = True self.chromePushed = False self.mozLogName = "moz.log" self.device = ADBDevice(adb=options.adbPath or 'adb', device=options.deviceSerial, test_root=options.remoteTestRoot, verbose=verbose) if options.remoteTestRoot is None: options.remoteTestRoot = self.device.test_root options.dumpOutputDirectory = options.remoteTestRoot self.remoteLogFile = posixpath.join(options.remoteTestRoot, "logs", "mochitest.log") logParent = posixpath.dirname(self.remoteLogFile) self.device.rm(logParent, force=True, recursive=True) self.device.mkdir(logParent) self.remoteProfile = posixpath.join(options.remoteTestRoot, "profile/") self.device.rm(self.remoteProfile, force=True, recursive=True) self.counts = dict() self.message_logger = MessageLogger(logger=None) self.message_logger.logger = self.log process_args = {'messageLogger': self.message_logger, 'counts': self.counts} self.automation = RemoteAutomation(self.device, options.remoteappname, self.remoteProfile, self.remoteLogFile, processArgs=process_args) self.environment = self.automation.environment # Check that Firefox is installed expected = options.app.split('/')[-1] if not self.device.is_app_installed(expected): raise Exception("%s is not installed on this device" % expected) self.automation.deleteANRs() self.automation.deleteTombstones() self.device.clear_logcat() self.remoteModulesDir = posixpath.join(options.remoteTestRoot, "modules/") self.remoteCache = posixpath.join(options.remoteTestRoot, "cache/") self.device.rm(self.remoteCache, force=True, recursive=True) # move necko cache to a location that can be cleaned up options.extraPrefs += ["browser.cache.disk.parent_directory=%s" % self.remoteCache] self.remoteMozLog = posixpath.join(options.remoteTestRoot, "mozlog") self.device.rm(self.remoteMozLog, force=True, recursive=True) self.device.mkdir(self.remoteMozLog) self.remoteChromeTestDir = posixpath.join( options.remoteTestRoot, "chrome") self.device.rm(self.remoteChromeTestDir, force=True, recursive=True) self.device.mkdir(self.remoteChromeTestDir) procName = options.app.split('/')[-1] self.device.stop_application(procName) if self.device.process_exist(procName): self.log.warning("unable to kill %s before running tests!" % procName) # Add Android version (SDK level) to mozinfo so that manifest entries # can be conditional on android_version. self.log.info( "Android sdk version '%s'; will use this to filter manifests" % str(self.device.version)) mozinfo.info['android_version'] = str(self.device.version) mozinfo.info['isFennec'] = not ('geckoview' in options.app) mozinfo.info['is_emulator'] = self.device._device_serial.startswith('emulator-') def cleanup(self, options, final=False): if final: self.device.rm(self.remoteChromeTestDir, force=True, recursive=True) self.chromePushed = False uploadDir = os.environ.get('MOZ_UPLOAD_DIR', None) if uploadDir and self.device.is_dir(self.remoteMozLog): self.device.pull(self.remoteMozLog, uploadDir) self.device.rm(self.remoteLogFile, force=True) self.device.rm(self.remoteProfile, force=True, recursive=True) self.device.rm(self.remoteCache, force=True, recursive=True) MochitestDesktop.cleanup(self, options, final) self.localProfile = None def dumpScreen(self, utilityPath): if self.haveDumpedScreen: self.log.info( "Not taking screenshot here: see the one that was previously logged") return self.haveDumpedScreen = True if self.device._device_serial.startswith('emulator-'): dump_screen(utilityPath, self.log) else: dump_device_screen(self.device, self.log) def findPath(self, paths, filename=None): for path in paths: p = path if filename: p = os.path.join(p, filename) if os.path.exists(self.getFullPath(p)): return path return None def makeLocalAutomation(self): localAutomation = Automation() localAutomation.IS_WIN32 = False localAutomation.IS_LINUX = False localAutomation.IS_MAC = False localAutomation.UNIXISH = False hostos = sys.platform if (hostos == 'mac' or hostos == 'darwin'): localAutomation.IS_MAC = True elif (hostos == 'linux' or hostos == 'linux2'): localAutomation.IS_LINUX = True localAutomation.UNIXISH = True elif (hostos == 'win32' or hostos == 'win64'): localAutomation.BIN_SUFFIX = ".exe" localAutomation.IS_WIN32 = True return localAutomation # This seems kludgy, but this class uses paths from the remote host in the # options, except when calling up to the base class, which doesn't # understand the distinction. This switches out the remote values for local # ones that the base class understands. This is necessary for the web # server, SSL tunnel and profile building functions. def switchToLocalPaths(self, options): """ Set local paths in the options, return a function that will restore remote values """ remoteXrePath = options.xrePath remoteProfilePath = options.profilePath remoteUtilityPath = options.utilityPath localAutomation = self.makeLocalAutomation() paths = [ options.xrePath, localAutomation.DIST_BIN, ] options.xrePath = self.findPath(paths) if options.xrePath is None: self.log.error( "unable to find xulrunner path for %s, please specify with --xre-path" % os.name) sys.exit(1) xpcshell = "xpcshell" if (os.name == "nt"): xpcshell += ".exe" if options.utilityPath: paths = [options.utilityPath, options.xrePath] else: paths = [options.xrePath] options.utilityPath = self.findPath(paths, xpcshell) if options.utilityPath is None: self.log.error( "unable to find utility path for %s, please specify with --utility-path" % os.name) sys.exit(1) xpcshell_path = os.path.join(options.utilityPath, xpcshell) if localAutomation.elf_arm(xpcshell_path): self.log.error('xpcshell at %s is an ARM binary; please use ' 'the --utility-path argument to specify the path ' 'to a desktop version.' % xpcshell_path) sys.exit(1) if self.localProfile: options.profilePath = self.localProfile else: options.profilePath = None def fixup(): options.xrePath = remoteXrePath options.utilityPath = remoteUtilityPath options.profilePath = remoteProfilePath return fixup def startServers(self, options, debuggerInfo): """ Create the servers on the host and start them up """ restoreRemotePaths = self.switchToLocalPaths(options) MochitestDesktop.startServers( self, options, debuggerInfo) restoreRemotePaths() def buildProfile(self, options): restoreRemotePaths = self.switchToLocalPaths(options) if options.testingModulesDir: try: self.device.push(options.testingModulesDir, self.remoteModulesDir) self.device.chmod(self.remoteModulesDir, recursive=True, root=True) except Exception: self.log.error( "Automation Error: Unable to copy test modules to device.") raise savedTestingModulesDir = options.testingModulesDir options.testingModulesDir = self.remoteModulesDir else: savedTestingModulesDir = None manifest = MochitestDesktop.buildProfile(self, options) if savedTestingModulesDir: options.testingModulesDir = savedTestingModulesDir self.localProfile = options.profilePath restoreRemotePaths() options.profilePath = self.remoteProfile return manifest def buildURLOptions(self, options, env): saveLogFile = options.logFile options.logFile = self.remoteLogFile options.profilePath = self.localProfile env["MOZ_HIDE_RESULTS_TABLE"] = "1" retVal = MochitestDesktop.buildURLOptions(self, options, env) # we really need testConfig.js (for browser chrome) try: self.device.push(options.profilePath, self.remoteProfile) self.device.chmod(self.remoteProfile, recursive=True, root=True) except Exception: self.log.error("Automation Error: Unable to copy profile to device.") raise options.profilePath = self.remoteProfile options.logFile = saveLogFile return retVal def getChromeTestDir(self, options): local = super(MochiRemote, self).getChromeTestDir(options) remote = self.remoteChromeTestDir if options.flavor == 'chrome' and not self.chromePushed: self.log.info("pushing %s to %s on device..." % (local, remote)) local = os.path.join(local, "chrome") self.device.push(local, remote) self.chromePushed = True return remote def getLogFilePath(self, logFile): return logFile def printDeviceInfo(self, printLogcat=False): try: if printLogcat: logcat = self.device.get_logcat( filter_out_regexps=fennecLogcatFilters) for l in logcat: ul = l.decode('utf-8', errors='replace') sl = ul.encode('iso8859-1', errors='replace') self.log.info(sl) self.log.info("Device info:") devinfo = self.device.get_info() for category in devinfo: if type(devinfo[category]) is list: self.log.info(" %s:" % category) for item in devinfo[category]: self.log.info(" %s" % item) else: self.log.info(" %s: %s" % (category, devinfo[category])) self.log.info("Test root: %s" % self.device.test_root) except ADBTimeoutError: raise except Exception as e: self.log.warning("Error getting device information: %s" % str(e)) def getGMPPluginPath(self, options): # TODO: bug 1149374 return None def buildBrowserEnv(self, options, debugger=False): browserEnv = MochitestDesktop.buildBrowserEnv( self, options, debugger=debugger) # remove desktop environment not used on device if "XPCOM_MEM_BLOAT_LOG" in browserEnv: del browserEnv["XPCOM_MEM_BLOAT_LOG"] # override mozLogs to avoid processing in MochitestDesktop base class self.mozLogs = None browserEnv["MOZ_LOG_FILE"] = os.path.join( self.remoteMozLog, self.mozLogName) if options.dmd: browserEnv['DMD'] = '1' # Contents of remoteMozLog will be pulled from device and copied to the # host MOZ_UPLOAD_DIR, to be made available as test artifacts. Make # MOZ_UPLOAD_DIR available to the browser environment so that tests # can use it as though they were running on the host. browserEnv["MOZ_UPLOAD_DIR"] = self.remoteMozLog return browserEnv def runApp(self, *args, **kwargs): """front-end automation.py's `runApp` functionality until FennecRunner is written""" # automation.py/remoteautomation `runApp` takes the profile path, # whereas runtest.py's `runApp` takes a mozprofile object. if 'profileDir' not in kwargs and 'profile' in kwargs: kwargs['profileDir'] = kwargs.pop('profile').profile # remove args not supported by automation.py kwargs.pop('marionette_args', None) ret, _ = self.automation.runApp(*args, **kwargs) self.countpass += self.counts['pass'] self.countfail += self.counts['fail'] self.counttodo += self.counts['todo'] return ret, None
def main(): automation = RemoteAutomation(None) parser = reftestcommandline.RemoteArgumentsParser() options = parser.parse_args() if (options.dm_trans == 'sut' and options.deviceIP == None): print "Error: If --dm_trans = sut, you must provide a device IP to connect to via the --deviceIP option" return 1 try: if (options.dm_trans == "adb"): if (options.deviceIP): dm = droid.DroidADB(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot) elif (options.deviceSerial): dm = droid.DroidADB(None, None, deviceSerial=options.deviceSerial, deviceRoot=options.remoteTestRoot) else: dm = droid.DroidADB(None, None, deviceRoot=options.remoteTestRoot) else: dm = droid.DroidSUT(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot) except devicemanager.DMError: print "Automation Error: exception while initializing devicemanager. Most likely the device is not in a testable state." return 1 automation.setDeviceManager(dm) if (options.remoteProductName != None): automation.setProduct(options.remoteProductName) # Set up the defaults and ensure options are set parser.validate_remote(options, automation) # Check that Firefox is installed expected = options.app.split('/')[-1] installed = dm.shellCheckOutput(['pm', 'list', 'packages', expected]) if expected not in installed: print "%s is not installed on this device" % expected return 1 automation.setAppName(options.app) automation.setRemoteProfile(options.remoteProfile) automation.setRemoteLog(options.remoteLogFile) reftest = RemoteReftest(automation, dm, options, SCRIPT_DIRECTORY) parser.validate(options, reftest) if mozinfo.info['debug']: print "changing timeout for remote debug reftests from %s to 600 seconds" % options.timeout options.timeout = 600 # Hack in a symbolic link for jsreftest os.system("ln -s ../jsreftest " + str(os.path.join(SCRIPT_DIRECTORY, "jsreftest"))) # Start the webserver retVal = reftest.startWebServer(options) if retVal: return retVal procName = options.app.split('/')[-1] if (dm.processExist(procName)): dm.killProcess(procName) reftest.printDeviceInfo() #an example manifest name to use on the cli # manifest = "http://" + options.remoteWebServer + "/reftests/layout/reftests/reftest-sanity/reftest.list" retVal = 0 try: dm.recordLogcat() retVal = reftest.runTests(options.tests, options) except: print "Automation Error: Exception caught while running tests" traceback.print_exc() retVal = 1 reftest.stopWebServer(options) reftest.printDeviceInfo(printLogcat=True) return retVal
def run_test_harness(parser, options): dm_args = { 'deviceRoot': options.remoteTestRoot, 'host': options.deviceIP, 'port': options.devicePort, } dm_args['adbPath'] = options.adb_path if not dm_args['host']: dm_args['deviceSerial'] = options.deviceSerial if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug': dm_args['logLevel'] = logging.DEBUG try: dm = mozdevice.DroidADB(**dm_args) except mozdevice.DMError: traceback.print_exc() print( "Automation Error: exception while initializing devicemanager. " "Most likely the device is not in a testable state.") return 1 automation = RemoteAutomation(None) automation.setDeviceManager(dm) if options.remoteProductName: automation.setProduct(options.remoteProductName) # Set up the defaults and ensure options are set parser.validate_remote(options, automation) # Check that Firefox is installed expected = options.app.split('/')[-1] installed = dm.shellCheckOutput(['pm', 'list', 'packages', expected]) if expected not in installed: print "%s is not installed on this device" % expected return 1 automation.setAppName(options.app) automation.setRemoteProfile(options.remoteProfile) automation.setRemoteLog(options.remoteLogFile) reftest = RemoteReftest(automation, dm, options, SCRIPT_DIRECTORY) parser.validate(options, reftest) if mozinfo.info['debug']: print "changing timeout for remote debug reftests from %s to 600 seconds" % options.timeout options.timeout = 600 # Hack in a symbolic link for jsreftest os.system("ln -s ../jsreftest " + str(os.path.join(SCRIPT_DIRECTORY, "jsreftest"))) # Despite our efforts to clean up servers started by this script, in practice # we still see infrequent cases where a process is orphaned and interferes # with future tests, typically because the old server is keeping the port in use. # Try to avoid those failures by checking for and killing servers before # trying to start new ones. reftest.killNamedProc('ssltunnel') reftest.killNamedProc('xpcshell') # Start the webserver retVal = reftest.startWebServer(options) if retVal: return retVal procName = options.app.split('/')[-1] dm.killProcess(procName) if dm.processExist(procName): print "unable to kill %s before starting tests!" % procName if options.printDeviceInfo: reftest.printDeviceInfo() # an example manifest name to use on the cli # manifest = "http://" + options.remoteWebServer + # "/reftests/layout/reftests/reftest-sanity/reftest.list" retVal = 0 try: dm.recordLogcat() retVal = reftest.runTests(options.tests, options) except: print "Automation Error: Exception caught while running tests" traceback.print_exc() retVal = 1 reftest.stopWebServer(options) if options.printDeviceInfo: reftest.printDeviceInfo(printLogcat=True) return retVal
def main(): scriptdir = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) auto = RemoteAutomation(None, "fennec") parser = RemoteOptions(auto, scriptdir) options, args = parser.parse_args() if (options.dm_trans == "adb"): if (options.deviceIP): dm = devicemanagerADB.DeviceManagerADB(options.deviceIP, options.devicePort) else: dm = devicemanagerADB.DeviceManagerADB() else: dm = devicemanagerSUT.DeviceManagerSUT(options.deviceIP, options.devicePort) auto.setDeviceManager(dm) options = parser.verifyRemoteOptions(options, auto) if (options == None): print "ERROR: Invalid options specified, use --help for a list of valid options" sys.exit(1) productPieces = options.remoteProductName.split('.') if (productPieces != None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) mochitest = MochiRemote(auto, dm, options) options = parser.verifyOptions(options, mochitest) if (options == None): sys.exit(1) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent) auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) print dm.getInfo() procName = options.app.split('/')[-1] if (dm.processExist(procName)): dm.killProcess(procName) if options.robocop != "": mp = manifestparser.TestManifest(strict=False) # TODO: pull this in dynamically mp.read(options.robocop) robocop_tests = mp.active_tests(exists=False) deviceRoot = dm.getDeviceRoot() dm.removeFile(os.path.join(deviceRoot, "fennec_ids.txt")) fennec_ids = os.path.abspath("fennec_ids.txt") if not os.path.exists(fennec_ids) and options.robocopIds: fennec_ids = options.robocopIds dm.pushFile(fennec_ids, os.path.join(deviceRoot, "fennec_ids.txt")) options.extraPrefs.append('robocop.logfile="%s/robocop.log"' % deviceRoot) options.extraPrefs.append('browser.search.suggest.enabled=true') options.extraPrefs.append('browser.search.suggest.prompted=true') if (options.dm_trans == 'adb' and options.robocopPath): dm._checkCmd([ "install", "-r", os.path.join(options.robocopPath, "robocop.apk") ]) appname = options.app retVal = None for test in robocop_tests: if options.testPath and options.testPath != test['name']: continue options.app = "am" options.browserArgs = [ "instrument", "-w", "-e", "deviceroot", deviceRoot, "-e", "class" ] options.browserArgs.append("%s.tests.%s" % (appname, test['name'])) options.browserArgs.append( "org.mozilla.roboexample.test/%s.FennecInstrumentationTestRunner" % appname) try: dm.recordLogcat() result = mochitest.runTests(options) if result != 0: print "ERROR: runTests() exited with code %s" % result # Ensure earlier failures aren't overwritten by success on this run if retVal is None or retVal == 0: retVal = result mochitest.addLogData() except: print "Automation Error: Exception caught while running tests" traceback.print_exc() mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) try: mochitest.cleanup(None, options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 break if retVal is None: print "No tests run. Did you pass an invalid TEST_PATH?" retVal = 1 else: # if we didn't have some kind of error running the tests, make # sure the tests actually passed overallResult = mochitest.printLog() if retVal == 0: retVal = overallResult else: try: dm.recordLogcat() retVal = mochitest.runTests(options) except: print "Automation Error: Exception caught while running tests" traceback.print_exc() mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) try: mochitest.cleanup(None, options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 try: logcat = dm.getLogcat(filterOutRegexps=fennecLogcatFilters) print ''.join(logcat) print dm.getInfo() except devicemanager.DMError: print "WARNING: Error getting device information at end of test" sys.exit(retVal)
def run_test_harness(options): message_logger = MessageLogger(logger=None) process_args = {'messageLogger': message_logger} auto = RemoteAutomation(None, "fennec", processArgs=process_args) if options is None: raise ValueError("Invalid options specified, use --help for a list of valid options") options.runByDir = False dm = options.dm auto.setDeviceManager(dm) mochitest = MochiRemote(auto, dm, options) log = mochitest.log message_logger.logger = log mochitest.message_logger = message_logger # Check that Firefox is installed expected = options.app.split('/')[-1] installed = dm.shellCheckOutput(['pm', 'list', 'packages', expected]) if expected not in installed: log.error("%s is not installed on this device" % expected) return 1 productPieces = options.remoteProductName.split('.') if (productPieces is not None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) auto.setAppName(options.remoteappname) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent) auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) if options.log_mach is None: mochitest.printDeviceInfo() # Add Android version (SDK level) to mozinfo so that manifest entries # can be conditional on android_version. androidVersion = dm.shellCheckOutput(['getprop', 'ro.build.version.sdk']) log.info( "Android sdk version '%s'; will use this to filter manifests" % str(androidVersion)) mozinfo.info['android_version'] = androidVersion deviceRoot = dm.deviceRoot if options.dmdPath: dmdLibrary = "libdmd.so" dmdPathOnDevice = os.path.join(deviceRoot, dmdLibrary) dm.removeFile(dmdPathOnDevice) dm.pushFile(os.path.join(options.dmdPath, dmdLibrary), dmdPathOnDevice) options.dmdPath = deviceRoot options.dumpOutputDirectory = deviceRoot procName = options.app.split('/')[-1] dm.killProcess(procName) mochitest.nsprLogName = "nspr.log" try: dm.recordLogcat() retVal = mochitest.runTests(options) except: log.error("Automation Error: Exception caught while running tests") traceback.print_exc() mochitest.stopServers() try: mochitest.cleanup(options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 if options.log_mach is None: mochitest.printDeviceInfo(printLogcat=True) message_logger.finish() return retVal
def run_test_harness(parser, options): dm_args = { 'deviceRoot': options.remoteTestRoot, 'host': options.deviceIP, 'port': options.devicePort, } dm_args['adbPath'] = options.adb_path if not dm_args['host']: dm_args['deviceSerial'] = options.deviceSerial try: dm = mozdevice.DroidADB(**dm_args) except mozdevice.DMError: traceback.print_exc() print( "Automation Error: exception while initializing devicemanager. " "Most likely the device is not in a testable state.") return 1 automation = RemoteAutomation(None) automation.setDeviceManager(dm) if options.remoteProductName: automation.setProduct(options.remoteProductName) # Set up the defaults and ensure options are set parser.validate_remote(options, automation) # Check that Firefox is installed expected = options.app.split('/')[-1] installed = dm.shellCheckOutput(['pm', 'list', 'packages', expected]) if expected not in installed: print "%s is not installed on this device" % expected return 1 automation.setAppName(options.app) automation.setRemoteProfile(options.remoteProfile) automation.setRemoteLog(options.remoteLogFile) reftest = RemoteReftest(automation, dm, options, SCRIPT_DIRECTORY) parser.validate(options, reftest) if mozinfo.info['debug']: print "changing timeout for remote debug reftests from %s to 600 seconds" % options.timeout options.timeout = 600 # Hack in a symbolic link for jsreftest os.system("ln -s ../jsreftest " + str(os.path.join(SCRIPT_DIRECTORY, "jsreftest"))) # Start the webserver retVal = reftest.startWebServer(options) if retVal: return retVal procName = options.app.split('/')[-1] if (dm.processExist(procName)): dm.killProcess(procName) if options.printDeviceInfo: reftest.printDeviceInfo() # an example manifest name to use on the cli # manifest = "http://" + options.remoteWebServer + # "/reftests/layout/reftests/reftest-sanity/reftest.list" retVal = 0 try: dm.recordLogcat() retVal = reftest.runTests(options.tests, options) except: print "Automation Error: Exception caught while running tests" traceback.print_exc() retVal = 1 reftest.stopWebServer(options) if options.printDeviceInfo: reftest.printDeviceInfo(printLogcat=True) return retVal
def main(): message_logger = MessageLogger(logger=None) process_args = {'messageLogger': message_logger} auto = RemoteAutomation(None, "fennec", processArgs=process_args) parser = RemoteOptions(auto) structured.commandline.add_logging_group(parser) options, args = parser.parse_args() if (options.dm_trans == "adb"): if (options.deviceIP): dm = droid.DroidADB(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot) else: dm = droid.DroidADB(deviceRoot=options.remoteTestRoot) else: dm = droid.DroidSUT(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot) auto.setDeviceManager(dm) options = parser.verifyRemoteOptions(options, auto) mochitest = MochiRemote(auto, dm, options) log = mochitest.log message_logger.logger = log mochitest.message_logger = message_logger if (options == None): log.error("Invalid options specified, use --help for a list of valid options") sys.exit(1) productPieces = options.remoteProductName.split('.') if (productPieces != None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) auto.setAppName(options.remoteappname) options = parser.verifyOptions(options, mochitest) if (options == None): sys.exit(1) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent); auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) mochitest.printDeviceInfo() # Add Android version (SDK level) to mozinfo so that manifest entries # can be conditional on android_version. androidVersion = dm.shellCheckOutput(['getprop', 'ro.build.version.sdk']) log.info("Android sdk version '%s'; will use this to filter manifests" % str(androidVersion)) mozinfo.info['android_version'] = androidVersion deviceRoot = dm.deviceRoot if options.dmdPath: dmdLibrary = "libdmd.so" dmdPathOnDevice = os.path.join(deviceRoot, dmdLibrary) dm.removeFile(dmdPathOnDevice) dm.pushFile(os.path.join(options.dmdPath, dmdLibrary), dmdPathOnDevice) options.dmdPath = deviceRoot options.dumpOutputDirectory = deviceRoot procName = options.app.split('/')[-1] dm.killProcess(procName) if options.robocopIni != "": # turning buffering off as it's not used in robocop message_logger.buffering = False # sut may wait up to 300 s for a robocop am process before returning dm.default_timeout = 320 mp = manifestparser.TestManifest(strict=False) # TODO: pull this in dynamically mp.read(options.robocopIni) robocop_tests = mp.active_tests(exists=False, **mozinfo.info) tests = [] my_tests = tests for test in robocop_tests: tests.append(test['name']) if options.totalChunks: tests_per_chunk = math.ceil(len(tests) / (options.totalChunks * 1.0)) start = int(round((options.thisChunk-1) * tests_per_chunk)) end = int(round(options.thisChunk * tests_per_chunk)) if end > len(tests): end = len(tests) my_tests = tests[start:end] log.info("Running tests %d-%d/%d" % (start+1, end, len(tests))) dm.removeFile(os.path.join(deviceRoot, "fennec_ids.txt")) fennec_ids = os.path.abspath(os.path.join(SCRIPT_DIR, "fennec_ids.txt")) if not os.path.exists(fennec_ids) and options.robocopIds: fennec_ids = options.robocopIds dm.pushFile(fennec_ids, os.path.join(deviceRoot, "fennec_ids.txt")) options.extraPrefs.append('browser.search.suggest.enabled=true') options.extraPrefs.append('browser.search.suggest.prompted=true') options.extraPrefs.append('layout.css.devPixelsPerPx=1.0') options.extraPrefs.append('browser.chrome.dynamictoolbar=false') options.extraPrefs.append('browser.snippets.enabled=false') options.extraPrefs.append('browser.casting.enabled=true') if (options.dm_trans == 'adb' and options.robocopApk): dm._checkCmd(["install", "-r", options.robocopApk]) retVal = None # Filtering tests active_tests = [] for test in robocop_tests: if options.testPath and options.testPath != test['name']: continue if not test['name'] in my_tests: continue if 'disabled' in test: log.info('TEST-INFO | skipping %s | %s' % (test['name'], test['disabled'])) continue active_tests.append(test) log.suite_start([t['name'] for t in active_tests]) for test in active_tests: # When running in a loop, we need to create a fresh profile for each cycle if mochitest.localProfile: options.profilePath = mochitest.localProfile os.system("rm -Rf %s" % options.profilePath) options.profilePath = None mochitest.localProfile = options.profilePath options.app = "am" options.browserArgs = ["instrument", "-w", "-e", "deviceroot", deviceRoot, "-e", "class"] options.browserArgs.append("org.mozilla.gecko.tests.%s" % test['name']) options.browserArgs.append("org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner") mochitest.nsprLogName = "nspr-%s.log" % test['name'] # If the test is for checking the import from bookmarks then make sure there is data to import if test['name'] == "testImportFromAndroid": # Get the OS so we can run the insert in the apropriate database and following the correct table schema osInfo = dm.getInfo("os") devOS = " ".join(osInfo['os']) if ("pandaboard" in devOS): delete = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"] else: delete = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"] if (options.dm_trans == "sut"): dm._runCmds([{"cmd": " ".join(delete)}]) # Insert the bookmarks log.info("Insert bookmarks in the default android browser database") for i in range(20): if ("pandaboard" in devOS): cmd = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser2.db 'insert or replace into bookmarks(_id,title,url,folder,parent,position) values (" + str(30 + i) + ",\"Bookmark"+ str(i) + "\",\"http://www.bookmark" + str(i) + ".com\",0,1," + str(100 + i) + ");'"] else: cmd = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser.db 'insert into bookmarks(title,url,bookmark) values (\"Bookmark"+ str(i) + "\",\"http://www.bookmark" + str(i) + ".com\",1);'"] if (options.dm_trans == "sut"): dm._runCmds([{"cmd": " ".join(cmd)}]) try: screenShotDir = "/mnt/sdcard/Robotium-Screenshots" dm.removeDir(screenShotDir) dm.recordLogcat() result = mochitest.runTests(options) if result != 0: log.error("runTests() exited with code %s" % result) log_result = mochitest.addLogData() if result != 0 or log_result != 0: mochitest.printDeviceInfo(printLogcat=True) mochitest.printScreenshots(screenShotDir) # Ensure earlier failures aren't overwritten by success on this run if retVal is None or retVal == 0: retVal = result except: log.error("Automation Error: Exception caught while running tests") traceback.print_exc() mochitest.stopServers() try: mochitest.cleanup(options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 break finally: # Clean-up added bookmarks if test['name'] == "testImportFromAndroid": if ("pandaboard" in devOS): cmd_del = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"] else: cmd_del = ['execsu', 'sqlite3', "/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"] if (options.dm_trans == "sut"): dm._runCmds([{"cmd": " ".join(cmd_del)}]) if retVal is None: log.warning("No tests run. Did you pass an invalid TEST_PATH?") retVal = 1 else: # if we didn't have some kind of error running the tests, make # sure the tests actually passed print "INFO | runtests.py | Test summary: start." overallResult = mochitest.printLog() print "INFO | runtests.py | Test summary: end." if retVal == 0: retVal = overallResult else: mochitest.nsprLogName = "nspr.log" try: dm.recordLogcat() retVal = mochitest.runTests(options) except: log.error("Automation Error: Exception caught while running tests") traceback.print_exc() mochitest.stopServers() try: mochitest.cleanup(options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 message_logger.finish() mochitest.printDeviceInfo(printLogcat=True) sys.exit(retVal)
def main(args): automation = RemoteAutomation(None) parser = RemoteOptions(automation) options, args = parser.parse_args() if (options.deviceIP == None): print "Error: you must provide a device IP to connect to via the --device option" return 1 try: if (options.dm_trans == "adb"): if (options.deviceIP): dm = droid.DroidADB(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot) else: dm = droid.DroidADB(None, None, deviceRoot=options.remoteTestRoot) else: dm = droid.DroidSUT(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot) except devicemanager.DMError: print "Error: exception while initializing devicemanager. Most likely the device is not in a testable state." return 1 automation.setDeviceManager(dm) if (options.remoteProductName != None): automation.setProduct(options.remoteProductName) # Set up the defaults and ensure options are set options = parser.verifyRemoteOptions(options) if (options == None): print "ERROR: Invalid options specified, use --help for a list of valid options" return 1 if not options.ignoreWindowSize: parts = dm.getInfo('screen')['screen'][0].split() width = int(parts[0].split(':')[1]) height = int(parts[1].split(':')[1]) if (width < 1050 or height < 1050): print "ERROR: Invalid screen resolution %sx%s, please adjust to 1366x1050 or higher" % ( width, height) return 1 automation.setAppName(options.app) automation.setRemoteProfile(options.remoteProfile) automation.setRemoteLog(options.remoteLogFile) reftest = RemoteReftest(automation, dm, options, SCRIPT_DIRECTORY) options = parser.verifyCommonOptions(options, reftest) # Hack in a symbolic link for jsreftest os.system("ln -s ../jsreftest " + str(os.path.join(SCRIPT_DIRECTORY, "jsreftest"))) # Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot manifest = args[0] if os.path.exists(os.path.join(SCRIPT_DIRECTORY, args[0])): manifest = "http://" + str(options.remoteWebServer) + ":" + str( options.httpPort) + "/" + args[0] elif os.path.exists(args[0]): manifestPath = os.path.abspath( args[0]).split(SCRIPT_DIRECTORY)[1].strip('/') manifest = "http://" + str(options.remoteWebServer) + ":" + str( options.httpPort) + "/" + manifestPath else: print "ERROR: Could not find test manifest '%s'" % manifest return 1 # Start the webserver retVal = reftest.startWebServer(options) if retVal: return retVal procName = options.app.split('/')[-1] if (dm.processExist(procName)): dm.killProcess(procName) reftest.printDeviceInfo() #an example manifest name to use on the cli # manifest = "http://" + options.remoteWebServer + "/reftests/layout/reftests/reftest-sanity/reftest.list" retVal = 0 try: cmdlineArgs = ["-reftest", manifest] if options.bootstrap: cmdlineArgs = [] dm.recordLogcat() retVal = reftest.runTests(manifest, options, cmdlineArgs) except: print "Automation Error: Exception caught while running tests" traceback.print_exc() retVal = 1 reftest.stopWebServer(options) reftest.printDeviceInfo(printLogcat=True) return retVal
def main(): scriptdir = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) auto = RemoteAutomation(None, "fennec") parser = RemoteOptions(auto, scriptdir) options, args = parser.parse_args() if (options.dm_trans == "adb"): if (options.deviceIP): dm = devicemanagerADB.DeviceManagerADB( options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot) else: dm = devicemanagerADB.DeviceManagerADB( deviceRoot=options.remoteTestRoot) else: dm = devicemanagerSUT.DeviceManagerSUT( options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot) auto.setDeviceManager(dm) options = parser.verifyRemoteOptions(options, auto) if (options == None): print "ERROR: Invalid options specified, use --help for a list of valid options" sys.exit(1) productPieces = options.remoteProductName.split('.') if (productPieces != None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) mochitest = MochiRemote(auto, dm, options) options = parser.verifyOptions(options, mochitest) if (options == None): sys.exit(1) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent) auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) print dm.getInfo() procName = options.app.split('/')[-1] if (dm.processExist(procName)): dm.killProcess(procName) if options.robocop != "": # sut may wait up to 300 s for a robocop am process before returning dm.default_timeout = 320 mp = manifestparser.TestManifest(strict=False) # TODO: pull this in dynamically mp.read(options.robocop) robocop_tests = mp.active_tests(exists=False) tests = [] my_tests = tests for test in robocop_tests: tests.append(test['name']) if options.totalChunks: tests_per_chunk = math.ceil( len(tests) / (options.totalChunks * 1.0)) start = int(round((options.thisChunk - 1) * tests_per_chunk)) end = int(round(options.thisChunk * tests_per_chunk)) if end > len(tests): end = len(tests) my_tests = tests[start:end] print "Running tests %d-%d/%d" % ((start + 1), end, len(tests)) deviceRoot = dm.getDeviceRoot() dm.removeFile(os.path.join(deviceRoot, "fennec_ids.txt")) fennec_ids = os.path.abspath("fennec_ids.txt") if not os.path.exists(fennec_ids) and options.robocopIds: fennec_ids = options.robocopIds dm.pushFile(fennec_ids, os.path.join(deviceRoot, "fennec_ids.txt")) options.extraPrefs.append('robocop.logfile="%s/robocop.log"' % deviceRoot) options.extraPrefs.append('browser.search.suggest.enabled=true') options.extraPrefs.append('browser.search.suggest.prompted=true') options.extraPrefs.append('browser.viewport.scaleRatio=100') options.extraPrefs.append('browser.chrome.dynamictoolbar=false') if (options.dm_trans == 'adb' and options.robocopPath): dm._checkCmd([ "install", "-r", os.path.join(options.robocopPath, "robocop.apk") ]) retVal = None for test in robocop_tests: if options.testPath and options.testPath != test['name']: continue if not test['name'] in my_tests: continue options.app = "am" options.browserArgs = [ "instrument", "-w", "-e", "deviceroot", deviceRoot, "-e", "class" ] options.browserArgs.append("%s.tests.%s" % (options.remoteappname, test['name'])) options.browserArgs.append( "org.mozilla.roboexample.test/%s.FennecInstrumentationTestRunner" % options.remoteappname) try: dm.removeDir("/mnt/sdcard/Robotium-Screenshots") dm.recordLogcat() result = mochitest.runTests(options) if result != 0: print "ERROR: runTests() exited with code %s" % result log_result = mochitest.addLogData() if result != 0 or log_result != 0: mochitest.printDeviceInfo() mochitest.printScreenshot() # Ensure earlier failures aren't overwritten by success on this run if retVal is None or retVal == 0: retVal = result except: print "Automation Error: Exception caught while running tests" traceback.print_exc() mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) try: mochitest.cleanup(None, options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 break if retVal is None: print "No tests run. Did you pass an invalid TEST_PATH?" retVal = 1 else: # if we didn't have some kind of error running the tests, make # sure the tests actually passed print "INFO | runtests.py | Test summary: start." overallResult = mochitest.printLog() print "INFO | runtests.py | Test summary: end." if retVal == 0: retVal = overallResult else: try: dm.recordLogcat() retVal = mochitest.runTests(options) except: print "Automation Error: Exception caught while running tests" traceback.print_exc() mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) try: mochitest.cleanup(None, options) except devicemanager.DMError: # device error cleaning up... oh well! pass retVal = 1 mochitest.printDeviceInfo() sys.exit(retVal)
def __init__(self, options): MochitestDesktop.__init__(self, options.flavor, vars(options)) verbose = False if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug': verbose = True if hasattr(options, 'log'): delattr(options, 'log') self.certdbNew = True self.chromePushed = False self.mozLogName = "moz.log" self.device = ADBDevice(adb=options.adbPath or 'adb', device=options.deviceSerial, test_root=options.remoteTestRoot, verbose=verbose) if options.remoteTestRoot is None: options.remoteTestRoot = self.device.test_root options.dumpOutputDirectory = options.remoteTestRoot self.remoteLogFile = posixpath.join(options.remoteTestRoot, "logs", "mochitest.log") logParent = posixpath.dirname(self.remoteLogFile) self.device.rm(logParent, force=True, recursive=True) self.device.mkdir(logParent) self.remoteProfile = posixpath.join(options.remoteTestRoot, "profile/") self.device.rm(self.remoteProfile, force=True, recursive=True) self.counts = dict() self.message_logger = MessageLogger(logger=None) self.message_logger.logger = self.log process_args = {'messageLogger': self.message_logger, 'counts': self.counts} self.automation = RemoteAutomation(self.device, options.remoteappname, self.remoteProfile, self.remoteLogFile, processArgs=process_args) self.environment = self.automation.environment # Check that Firefox is installed expected = options.app.split('/')[-1] if not self.device.is_app_installed(expected): raise Exception("%s is not installed on this device" % expected) self.automation.deleteANRs() self.automation.deleteTombstones() self.device.clear_logcat() self.remoteModulesDir = posixpath.join(options.remoteTestRoot, "modules/") self.remoteCache = posixpath.join(options.remoteTestRoot, "cache/") self.device.rm(self.remoteCache, force=True, recursive=True) # move necko cache to a location that can be cleaned up options.extraPrefs += ["browser.cache.disk.parent_directory=%s" % self.remoteCache] self.remoteMozLog = posixpath.join(options.remoteTestRoot, "mozlog") self.device.rm(self.remoteMozLog, force=True, recursive=True) self.device.mkdir(self.remoteMozLog) self.remoteChromeTestDir = posixpath.join( options.remoteTestRoot, "chrome") self.device.rm(self.remoteChromeTestDir, force=True, recursive=True) self.device.mkdir(self.remoteChromeTestDir) procName = options.app.split('/')[-1] self.device.stop_application(procName) if self.device.process_exist(procName): self.log.warning("unable to kill %s before running tests!" % procName) # Add Android version (SDK level) to mozinfo so that manifest entries # can be conditional on android_version. self.log.info( "Android sdk version '%s'; will use this to filter manifests" % str(self.device.version)) mozinfo.info['android_version'] = str(self.device.version) mozinfo.info['isFennec'] = not ('geckoview' in options.app) mozinfo.info['is_emulator'] = self.device._device_serial.startswith('emulator-')
def run_test_harness(parser, options): dm_args = { 'deviceRoot': options.remoteTestRoot, 'host': options.deviceIP, 'port': options.devicePort, } dm_args['adbPath'] = options.adb_path if not dm_args['host']: dm_args['deviceSerial'] = options.deviceSerial if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug': dm_args['logLevel'] = logging.DEBUG try: dm = mozdevice.DroidADB(**dm_args) except mozdevice.DMError: traceback.print_exc() print ("Automation Error: exception while initializing devicemanager. " "Most likely the device is not in a testable state.") return 1 automation = RemoteAutomation(None) automation.setDeviceManager(dm) if options.remoteProductName: automation.setProduct(options.remoteProductName) # Set up the defaults and ensure options are set parser.validate_remote(options, automation) # Check that Firefox is installed expected = options.app.split('/')[-1] installed = dm.shellCheckOutput(['pm', 'list', 'packages', expected]) if expected not in installed: print "%s is not installed on this device" % expected return 1 automation.setAppName(options.app) automation.setRemoteProfile(options.remoteProfile) automation.setRemoteLog(options.remoteLogFile) reftest = RemoteReftest(automation, dm, options, SCRIPT_DIRECTORY) parser.validate(options, reftest) if mozinfo.info['debug']: print "changing timeout for remote debug reftests from %s to 600 seconds" % options.timeout options.timeout = 600 # Hack in a symbolic link for jsreftest os.system("ln -s ../jsreftest " + str(os.path.join(SCRIPT_DIRECTORY, "jsreftest"))) # Start the webserver retVal = reftest.startWebServer(options) if retVal: return retVal procName = options.app.split('/')[-1] dm.killProcess(procName) if dm.processExist(procName): print "unable to kill %s before starting tests!" % procName if options.printDeviceInfo: reftest.printDeviceInfo() # an example manifest name to use on the cli # manifest = "http://" + options.remoteWebServer + # "/reftests/layout/reftests/reftest-sanity/reftest.list" retVal = 0 try: dm.recordLogcat() retVal = reftest.runTests(options.tests, options) except: print "Automation Error: Exception caught while running tests" traceback.print_exc() retVal = 1 reftest.stopWebServer(options) if options.printDeviceInfo: reftest.printDeviceInfo(printLogcat=True) return retVal
def main(): scriptdir = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) dm_none = devicemanagerADB.DeviceManagerADB() auto = RemoteAutomation(dm_none, "fennec") parser = RemoteOptions(auto, scriptdir) options, args = parser.parse_args() if (options.dm_trans == "adb"): if (options.deviceIP): dm = devicemanagerADB.DeviceManagerADB(options.deviceIP, options.devicePort) else: dm = dm_none else: dm = devicemanagerSUT.DeviceManagerSUT(options.deviceIP, options.devicePort) auto.setDeviceManager(dm) options = parser.verifyRemoteOptions(options, auto) if (options == None): print "ERROR: Invalid options specified, use --help for a list of valid options" sys.exit(1) productPieces = options.remoteProductName.split('.') if (productPieces != None): auto.setProduct(productPieces[0]) else: auto.setProduct(options.remoteProductName) mochitest = MochiRemote(auto, dm, options) options = parser.verifyOptions(options, mochitest) if (options == None): sys.exit(1) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent); auto.setRemoteLog(options.remoteLogFile) auto.setServerInfo(options.webServer, options.httpPort, options.sslPort) procName = options.app.split('/')[-1] if (dm.processExist(procName)): dm.killProcess(procName) if options.robocop != "": mp = manifestparser.TestManifest(strict=False) # TODO: pull this in dynamically mp.read(options.robocop) robocop_tests = mp.active_tests(exists=False) fHandle = open("robotium.config", "w") fHandle.write("profile=%s\n" % (mochitest.remoteProfile)) fHandle.write("logfile=%s\n" % (options.remoteLogFile)) fHandle.close() deviceRoot = dm.getDeviceRoot() # Note, we are pushing to /sdcard since we have this location hard coded in robocop dm.removeFile("/sdcard/fennec_ids.txt") dm.removeFile("/sdcard/robotium.config") dm.pushFile("robotium.config", "/sdcard/robotium.config") fennec_ids = os.path.abspath("fennec_ids.txt") if not os.path.exists(fennec_ids) and options.robocopPath: fennec_ids = os.path.abspath(os.path.join(options.robocopPath, "fennec_ids.txt")) dm.pushFile(fennec_ids, "/sdcard/fennec_ids.txt") options.extraPrefs.append('robocop.logfile="%s/robocop.log"' % deviceRoot) if (options.dm_trans == 'adb' and options.robocopPath): dm.checkCmd(["install", "-r", os.path.join(options.robocopPath, "robocop.apk")]) appname = options.app for test in robocop_tests: if options.testPath and options.testPath != test['name']: continue options.app = "am" options.browserArgs = ["instrument", "-w", "-e", "class"] options.browserArgs.append("%s.tests.%s" % (appname, test['name'])) options.browserArgs.append("org.mozilla.roboexample.test/android.test.InstrumentationTestRunner") try: retVal = mochitest.runTests(options) except: print "TEST-UNEXPECTED-ERROR | %s | Exception caught while running robocop tests." % sys.exc_info()[1] mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) try: self.cleanup(None, options) except: pass sys.exit(1) else: try: retVal = mochitest.runTests(options) except: print "TEST-UNEXPECTED-ERROR | %s | Exception caught while running tests." % sys.exc_info()[1] mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) try: self.cleanup(None, options) except: pass sys.exit(1) sys.exit(retVal)