class MarionetteTestRunner(object): textrunnerclass = MarionetteTextTestRunner def __init__(self, address=None, emulator=None, emulatorBinary=None, emulatorImg=None, emulator_res='480x800', homedir=None, app=None, bin=None, profile=None, autolog=False, revision=None, logger=None, testgroup="marionette", noWindow=False, logcat_dir=None, xml_output=None, repeat=0, gecko_path=None, testvars=None, tree=None, type=None, device_serial=None, symbols_path=None, timeout=None, es_servers=None, **kwargs): self.address = address self.emulator = emulator self.emulatorBinary = emulatorBinary self.emulatorImg = emulatorImg self.emulator_res = emulator_res self.homedir = homedir self.app = app self.bin = bin self.profile = profile self.autolog = autolog self.testgroup = testgroup self.revision = revision self.logger = logger self.noWindow = noWindow self.httpd = None self.baseurl = None self.marionette = None self.logcat_dir = logcat_dir self.xml_output = xml_output self.repeat = repeat self.gecko_path = gecko_path self.testvars = {} self.test_kwargs = kwargs self.tree = tree self.type = type self.device_serial = device_serial self.symbols_path = symbols_path self.timeout = timeout self._device = None self._capabilities = None self._appName = None self.es_servers = es_servers if testvars: if not os.path.exists(testvars): raise Exception('--testvars file does not exist') import json with open(testvars) as f: self.testvars = json.loads(f.read()) # set up test handlers self.test_handlers = [] self.register_handlers() self.reset_test_stats() if self.logger is None: self.logger = logging.getLogger('Marionette') self.logger.setLevel(logging.INFO) self.logger.addHandler(logging.StreamHandler()) if self.logcat_dir: if not os.access(self.logcat_dir, os.F_OK): os.mkdir(self.logcat_dir) # for XML output self.testvars['xml_output'] = self.xml_output self.results = [] @property def capabilities(self): if self._capabilities: return self._capabilities self.marionette.start_session() self._capabilities = self.marionette.session_capabilities self.marionette.delete_session() return self._capabilities @property def device(self): if self._device: return self._device self._device = self.capabilities.get('device') return self._device @property def appName(self): if self._appName: return self._appName self._appName = self.capabilities.get('browserName') return self._appName def reset_test_stats(self): self.passed = 0 self.failed = 0 self.todo = 0 self.failures = [] def start_httpd(self): host = moznetwork.get_ip() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("",0)) port = s.getsockname()[1] s.close() self.baseurl = 'http://%s:%d/' % (host, port) self.logger.info('running webserver on %s' % self.baseurl) self.httpd = MozHttpd(host=host, port=port, docroot=os.path.join(os.path.dirname(__file__), 'www')) self.httpd.start() def start_marionette(self): assert(self.baseurl is not None) if self.bin: if self.address: host, port = self.address.split(':') else: host = 'localhost' port = 2828 self.marionette = Marionette(host=host, port=int(port), app=self.app, bin=self.bin, profile=self.profile, baseurl=self.baseurl, timeout=self.timeout) elif self.address: host, port = self.address.split(':') try: #establish a telnet connection so we can vertify the data come back tlconnection = Telnet(host, port) except: raise Exception("could not connect to given marionette host/port") if self.emulator: self.marionette = Marionette.getMarionetteOrExit( host=host, port=int(port), connectToRunningEmulator=True, homedir=self.homedir, baseurl=self.baseurl, logcat_dir=self.logcat_dir, gecko_path=self.gecko_path, symbols_path=self.symbols_path, timeout=self.timeout) else: self.marionette = Marionette(host=host, port=int(port), baseurl=self.baseurl, timeout=self.timeout) elif self.emulator: self.marionette = Marionette.getMarionetteOrExit( emulator=self.emulator, emulatorBinary=self.emulatorBinary, emulatorImg=self.emulatorImg, emulator_res=self.emulator_res, homedir=self.homedir, baseurl=self.baseurl, noWindow=self.noWindow, logcat_dir=self.logcat_dir, gecko_path=self.gecko_path, symbols_path=self.symbols_path, timeout=self.timeout) else: raise Exception("must specify binary, address or emulator") def post_to_autolog(self, elapsedtime): self.logger.info('posting results to autolog') logfile = None if self.emulator: filename = os.path.join(os.path.abspath(self.logcat_dir), "emulator-%d.log" % self.marionette.emulator.port) if os.access(filename, os.F_OK): logfile = filename for es_server in self.es_servers: # This is all autolog stuff. # See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog from mozautolog import RESTfulAutologTestGroup testgroup = RESTfulAutologTestGroup( testgroup=self.testgroup, os='android', platform='emulator', harness='marionette', server=es_server, restserver=None, machine=socket.gethostname(), logfile=logfile) testgroup.set_primary_product( tree=self.tree, buildtype='opt', revision=self.revision) testgroup.add_test_suite( testsuite='b2g emulator testsuite', elapsedtime=elapsedtime.seconds, cmdline='', passed=self.passed, failed=self.failed, todo=self.todo) # Add in the test failures. for f in self.failures: testgroup.add_test_failure(test=f[0], text=f[1], status=f[2]) testgroup.submit() def run_tests(self, tests): self.reset_test_stats() starttime = datetime.utcnow() while self.repeat >=0: for test in tests: self.run_test(test) self.repeat -= 1 self.logger.info('\nSUMMARY\n-------') self.logger.info('passed: %d' % self.passed) self.logger.info('failed: %d' % self.failed) self.logger.info('todo: %d' % self.todo) if self.failed > 0: self.logger.info('\nFAILED TESTS\n-------') for failed_test in self.failures: self.logger.info('%s' % failed_test[0]) try: self.marionette.check_for_crash() except: traceback.print_exc() self.elapsedtime = datetime.utcnow() - starttime if self.autolog: self.post_to_autolog(self.elapsedtime) if self.xml_output: xml_dir = os.path.dirname(os.path.abspath(self.xml_output)) if not os.path.exists(xml_dir): os.makedirs(xml_dir) with open(self.xml_output, 'w') as f: f.write(self.generate_xml(self.results)) if self.marionette.instance: self.marionette.instance.close() self.marionette.instance = None del self.marionette def run_test(self, test): if not self.httpd: print "starting httpd" self.start_httpd() if not self.marionette: self.start_marionette() filepath = os.path.abspath(test) if os.path.isdir(filepath): for root, dirs, files in os.walk(filepath): for filename in files: if ((filename.startswith('test_') or filename.startswith('browser_')) and (filename.endswith('.py') or filename.endswith('.js'))): filepath = os.path.join(root, filename) self.run_test(filepath) if self.marionette.check_for_crash(): return return mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1]) testloader = unittest.TestLoader() suite = unittest.TestSuite() if file_ext == '.ini': testargs = {} if self.type is not None: testtypes = self.type.replace('+', ' +').replace('-', ' -').split() for atype in testtypes: if atype.startswith('+'): testargs.update({ atype[1:]: 'true' }) elif atype.startswith('-'): testargs.update({ atype[1:]: 'false' }) else: testargs.update({ atype: 'true' }) manifest = TestManifest() manifest.read(filepath) all_tests = manifest.active_tests(disabled=False) manifest_tests = manifest.active_tests(disabled=False, device=self.device, app=self.appName) skip_tests = list(set([x['path'] for x in all_tests]) - set([x['path'] for x in manifest_tests])) for skipped in skip_tests: self.logger.info('TEST-SKIP | %s | device=%s, app=%s' % (os.path.basename(skipped), self.device, self.appName)) self.todo += 1 for i in manifest.get(tests=manifest_tests, **testargs): self.run_test(i["path"]) if self.marionette.check_for_crash(): return return self.logger.info('TEST-START %s' % os.path.basename(test)) for handler in self.test_handlers: if handler.match(os.path.basename(test)): handler.add_tests_to_suite(mod_name, filepath, suite, testloader, self.marionette, self.testvars, **self.test_kwargs) break if suite.countTestCases(): runner = self.textrunnerclass(verbosity=3, marionette=self.marionette) results = runner.run(suite) self.results.append(results) self.failed += len(results.failures) + len(results.errors) if hasattr(results, 'skipped'): self.todo += len(results.skipped) self.passed += results.passed for failure in results.failures + results.errors: self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL')) if hasattr(results, 'unexpectedSuccesses'): self.failed += len(results.unexpectedSuccesses) for failure in results.unexpectedSuccesses: self.failures.append((results.getInfo(failure), 'TEST-UNEXPECTED-PASS')) if hasattr(results, 'expectedFailures'): self.passed += len(results.expectedFailures) def register_handlers(self): self.test_handlers.extend([MarionetteTestCase, MarionetteJSTestCase]) def cleanup(self): if self.httpd: self.httpd.stop() __del__ = cleanup def generate_xml(self, results_list): def _extract_xml(test, text='', result='passed'): cls_name = test.__class__.__name__ testcase = doc.createElement('testcase') testcase.setAttribute('classname', cls_name) testcase.setAttribute('name', unicode(test).split()[0]) testcase.setAttribute('time', str(test.duration)) testsuite.appendChild(testcase) if result in ['failure', 'error', 'skipped']: f = doc.createElement(result) f.setAttribute('message', 'test %s' % result) f.appendChild(doc.createTextNode(text)) testcase.appendChild(f) doc = dom.Document() testsuite = doc.createElement('testsuite') testsuite.setAttribute('name', 'Marionette') testsuite.setAttribute('time', str(self.elapsedtime.total_seconds())) testsuite.setAttribute('tests', str(sum([results.testsRun for results in results_list]))) def failed_count(results): count = len(results.failures) if hasattr(results, 'unexpectedSuccesses'): count += len(results.unexpectedSuccesses) return count testsuite.setAttribute('failures', str(sum([failed_count(results) for results in results_list]))) testsuite.setAttribute('errors', str(sum([len(results.errors) for results in results_list]))) if hasattr(results, 'skipped'): testsuite.setAttribute('skips', str(sum([len(results.skipped) + len(results.expectedFailures) for results in results_list]))) for results in results_list: for tup in results.errors: _extract_xml(tup[0], text=tup[1], result='error') for tup in results.failures: _extract_xml(tup[0], text=tup[1], result='failure') if hasattr(results, 'unexpectedSuccesses'): for test in results.unexpectedSuccesses: # unexpectedSuccesses is a list of Testcases only, no tuples _extract_xml(test, text='TEST-UNEXPECTED-PASS', result='failure') if hasattr(results, 'skipped'): for tup in results.skipped: _extract_xml(tup[0], text=tup[1], result='skipped') if hasattr(results, 'expectedFailures'): for tup in results.expectedFailures: _extract_xml(tup[0], text=tup[1], result='skipped') for test in results.tests_passed: _extract_xml(test) doc.appendChild(testsuite) return doc.toprettyxml(encoding='utf-8')
class MarionetteTestRunner(object): def __init__(self, address=None, emulator=None, emulatorBinary=None, emulatorImg=None, emulator_res='480x800', homedir=None, app=None, bin=None, profile=None, autolog=False, revision=None, es_server=None, rest_server=None, logger=None, testgroup="marionette", noWindow=False, logcat_dir=None, xml_output=None, repeat=0, gecko_path=None, testvars=None, tree=None, type=None, device=None, symbols_path=None, **kwargs): self.address = address self.emulator = emulator self.emulatorBinary = emulatorBinary self.emulatorImg = emulatorImg self.emulator_res = emulator_res self.homedir = homedir self.app = app self.bin = bin self.profile = profile self.autolog = autolog self.testgroup = testgroup self.revision = revision self.es_server = es_server self.rest_server = rest_server self.logger = logger self.noWindow = noWindow self.httpd = None self.baseurl = None self.marionette = None self.logcat_dir = logcat_dir self.xml_output = xml_output self.repeat = repeat self.gecko_path = gecko_path self.testvars = {} self.test_kwargs = kwargs self.tree = tree self.type = type self.device = device self.symbols_path = symbols_path if testvars: if not os.path.exists(testvars): raise Exception('--testvars file does not exist') import json with open(testvars) as f: self.testvars = json.loads(f.read()) # set up test handlers self.test_handlers = [] self.register_handlers() self.reset_test_stats() if self.logger is None: self.logger = logging.getLogger('Marionette') self.logger.setLevel(logging.INFO) self.logger.addHandler(logging.StreamHandler()) if self.logcat_dir: if not os.access(self.logcat_dir, os.F_OK): os.mkdir(self.logcat_dir) # for XML output self.testvars['xml_output'] = self.xml_output self.results = [] def reset_test_stats(self): self.passed = 0 self.failed = 0 self.todo = 0 self.failures = [] def start_httpd(self): host = moznetwork.get_ip() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("", 0)) port = s.getsockname()[1] s.close() self.baseurl = 'http://%s:%d/' % (host, port) self.logger.info('running webserver on %s' % self.baseurl) self.httpd = MozHttpd(host=host, port=port, docroot=os.path.join(os.path.dirname(__file__), 'www')) self.httpd.start() def start_marionette(self): assert (self.baseurl is not None) if self.bin: if self.address: host, port = self.address.split(':') else: host = 'localhost' port = 2828 self.marionette = Marionette(host=host, port=int(port), app=self.app, bin=self.bin, profile=self.profile, baseurl=self.baseurl) elif self.address: host, port = self.address.split(':') if self.emulator: self.marionette = Marionette.getMarionetteOrExit( host=host, port=int(port), connectToRunningEmulator=True, homedir=self.homedir, baseurl=self.baseurl, logcat_dir=self.logcat_dir, gecko_path=self.gecko_path, symbols_path=self.symbols_path) else: self.marionette = Marionette(host=host, port=int(port), baseurl=self.baseurl) elif self.emulator: self.marionette = Marionette.getMarionetteOrExit( emulator=self.emulator, emulatorBinary=self.emulatorBinary, emulatorImg=self.emulatorImg, emulator_res=self.emulator_res, homedir=self.homedir, baseurl=self.baseurl, noWindow=self.noWindow, logcat_dir=self.logcat_dir, gecko_path=self.gecko_path, symbols_path=self.symbols_path) else: raise Exception("must specify binary, address or emulator") def post_to_autolog(self, elapsedtime): self.logger.info('posting results to autolog') logfile = None if self.emulator: filename = os.path.join( os.path.abspath(self.logcat_dir), "emulator-%d.log" % self.marionette.emulator.port) if os.access(filename, os.F_OK): logfile = filename # This is all autolog stuff. # See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog from mozautolog import RESTfulAutologTestGroup testgroup = RESTfulAutologTestGroup(testgroup=self.testgroup, os='android', platform='emulator', harness='marionette', server=self.es_server, restserver=self.rest_server, machine=socket.gethostname(), logfile=logfile) testgroup.set_primary_product(tree=self.tree, buildtype='opt', revision=self.revision) testgroup.add_test_suite(testsuite='b2g emulator testsuite', elapsedtime=elapsedtime.seconds, cmdline='', passed=self.passed, failed=self.failed, todo=self.todo) # Add in the test failures. for f in self.failures: testgroup.add_test_failure(test=f[0], text=f[1], status=f[2]) testgroup.submit() def run_tests(self, tests): self.reset_test_stats() starttime = datetime.utcnow() while self.repeat >= 0: for test in tests: self.run_test(test) self.repeat -= 1 self.logger.info('\nSUMMARY\n-------') self.logger.info('passed: %d' % self.passed) self.logger.info('failed: %d' % self.failed) self.logger.info('todo: %d' % self.todo) try: self.marionette.check_for_crash() except: traceback.print_exc() self.elapsedtime = datetime.utcnow() - starttime if self.autolog: self.post_to_autolog(self.elapsedtime) if self.xml_output: xml_dir = os.path.dirname(os.path.abspath(self.xml_output)) if not os.path.exists(xml_dir): os.makedirs(xml_dir) with open(self.xml_output, 'w') as f: f.write(self.generate_xml(self.results)) if self.marionette.instance: self.marionette.instance.close() self.marionette.instance = None del self.marionette def run_test(self, test): if not self.httpd: print "starting httpd" self.start_httpd() if not self.marionette: self.start_marionette() filepath = os.path.abspath(test) if os.path.isdir(filepath): for root, dirs, files in os.walk(filepath): for filename in files: if ((filename.startswith('test_') or filename.startswith('browser_')) and (filename.endswith('.py') or filename.endswith('.js'))): filepath = os.path.join(root, filename) self.run_test(filepath) if self.marionette.check_for_crash(): return return mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1]) testloader = unittest.TestLoader() suite = unittest.TestSuite() if file_ext == '.ini': testargs = {} if self.type is not None: testtypes = self.type.replace('+', ' +').replace('-', ' -').split() for atype in testtypes: if atype.startswith('+'): testargs.update({atype[1:]: 'true'}) elif atype.startswith('-'): testargs.update({atype[1:]: 'false'}) else: testargs.update({atype: 'true'}) manifest = TestManifest() manifest.read(filepath) manifest_tests = manifest.active_tests(disabled=False) for i in manifest.get(tests=manifest_tests, **testargs): self.run_test(i["path"]) if self.marionette.check_for_crash(): return return self.logger.info('TEST-START %s' % os.path.basename(test)) for handler in self.test_handlers: if handler.match(os.path.basename(test)): handler.add_tests_to_suite(mod_name, filepath, suite, testloader, self.marionette, self.testvars, **self.test_kwargs) break if suite.countTestCases(): runner = MarionetteTextTestRunner(verbosity=3, marionette=self.marionette) results = runner.run(suite) self.results.append(results) self.failed += len(results.failures) + len(results.errors) if hasattr(results, 'skipped'): self.todo += len(results.skipped) + len( results.expectedFailures) self.passed += results.passed for failure in results.failures + results.errors: self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL')) if hasattr(results, 'unexpectedSuccess'): self.failed += len(results.unexpectedSuccesses) for failure in results.unexpectedSuccesses: self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-PASS')) def register_handlers(self): self.test_handlers.extend([MarionetteTestCase, MarionetteJSTestCase]) def cleanup(self): if self.httpd: self.httpd.stop() __del__ = cleanup def generate_xml(self, results_list): def _extract_xml(test, text='', result='passed'): cls_name = test.__class__.__name__ testcase = doc.createElement('testcase') testcase.setAttribute('classname', cls_name) testcase.setAttribute('name', unicode(test).split()[0]) testcase.setAttribute('time', str(test.duration)) testsuite.appendChild(testcase) if result in ['failure', 'error', 'skipped']: f = doc.createElement(result) f.setAttribute('message', 'test %s' % result) f.appendChild(doc.createTextNode(text)) testcase.appendChild(f) doc = dom.Document() testsuite = doc.createElement('testsuite') testsuite.setAttribute('name', 'Marionette') testsuite.setAttribute('time', str(self.elapsedtime.total_seconds())) testsuite.setAttribute( 'tests', str(sum([results.testsRun for results in results_list]))) def failed_count(results): count = len(results.failures) if hasattr(results, 'unexpectedSuccesses'): count += len(results.unexpectedSuccesses) return count testsuite.setAttribute( 'failures', str(sum([failed_count(results) for results in results_list]))) testsuite.setAttribute( 'errors', str(sum([len(results.errors) for results in results_list]))) if hasattr(results, 'skipped'): testsuite.setAttribute( 'skips', str( sum([ len(results.skipped) + len(results.expectedFailures) for results in results_list ]))) for results in results_list: for tup in results.errors: _extract_xml(*tup, result='error') for tup in results.failures: _extract_xml(*tup, result='failure') if hasattr(results, 'unexpectedSuccesses'): for test in results.unexpectedSuccesses: # unexpectedSuccesses is a list of Testcases only, no tuples _extract_xml(test, text='TEST-UNEXPECTED-PASS', result='failure') if hasattr(results, 'skipped'): for tup in results.skipped: _extract_xml(*tup, result='skipped') if hasattr(results, 'expectedFailures'): for tup in results.expectedFailures: _extract_xml(*tup, result='skipped') for test in results.tests_passed: _extract_xml(test) doc.appendChild(testsuite) return doc.toprettyxml(encoding='utf-8')
class MarionetteTestRunner(object): def __init__( self, address=None, emulator=None, emulatorBinary=None, emulatorImg=None, emulator_res="480x800", homedir=None, app=None, bin=None, profile=None, autolog=False, revision=None, es_server=None, rest_server=None, logger=None, testgroup="marionette", noWindow=False, logcat_dir=None, xml_output=None, repeat=0, perf=False, perfserv=None, gecko_path=None, testvars=None, tree=None, device=None, symbols_path=None, ): self.address = address self.emulator = emulator self.emulatorBinary = emulatorBinary self.emulatorImg = emulatorImg self.emulator_res = emulator_res self.homedir = homedir self.app = app self.bin = bin self.profile = profile self.autolog = autolog self.testgroup = testgroup self.revision = revision self.es_server = es_server self.rest_server = rest_server self.logger = logger self.noWindow = noWindow self.httpd = None self.baseurl = None self.marionette = None self.logcat_dir = logcat_dir self.perfrequest = None self.xml_output = xml_output self.repeat = repeat self.perf = perf self.perfserv = perfserv self.gecko_path = gecko_path self.testvars = {} self.tree = tree self.device = device self.symbols_path = symbols_path if testvars: if not os.path.exists(testvars): raise Exception("--testvars file does not exist") import json with open(testvars) as f: self.testvars = json.loads(f.read()) # set up test handlers self.test_handlers = [] self.register_handlers() self.reset_test_stats() if self.logger is None: self.logger = logging.getLogger("Marionette") self.logger.setLevel(logging.INFO) self.logger.addHandler(logging.StreamHandler()) if self.logcat_dir: if not os.access(self.logcat_dir, os.F_OK): os.mkdir(self.logcat_dir) # for XML output self.testvars["xml_output"] = self.xml_output self.results = [] def reset_test_stats(self): self.passed = 0 self.failed = 0 self.todo = 0 self.failures = [] self.perfrequest = None def start_httpd(self): host = moznetwork.get_ip() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("", 0)) port = s.getsockname()[1] s.close() self.baseurl = "http://%s:%d/" % (host, port) self.logger.info("running webserver on %s" % self.baseurl) self.httpd = MozHttpd(host=host, port=port, docroot=os.path.join(os.path.dirname(__file__), "www")) self.httpd.start() def start_marionette(self): assert self.baseurl is not None if self.bin: if self.address: host, port = self.address.split(":") else: host = "localhost" port = 2828 self.marionette = Marionette( host=host, port=int(port), app=self.app, bin=self.bin, profile=self.profile, baseurl=self.baseurl ) elif self.address: host, port = self.address.split(":") if self.emulator: self.marionette = Marionette.getMarionetteOrExit( host=host, port=int(port), connectToRunningEmulator=True, homedir=self.homedir, baseurl=self.baseurl, logcat_dir=self.logcat_dir, gecko_path=self.gecko_path, symbols_path=self.symbols_path, ) else: self.marionette = Marionette(host=host, port=int(port), baseurl=self.baseurl) elif self.emulator: self.marionette = Marionette.getMarionetteOrExit( emulator=self.emulator, emulatorBinary=self.emulatorBinary, emulatorImg=self.emulatorImg, emulator_res=self.emulator_res, homedir=self.homedir, baseurl=self.baseurl, noWindow=self.noWindow, logcat_dir=self.logcat_dir, gecko_path=self.gecko_path, symbols_path=self.symbols_path, ) else: raise Exception("must specify binary, address or emulator") def post_to_autolog(self, elapsedtime): self.logger.info("posting results to autolog") logfile = None if self.emulator: filename = os.path.join(os.path.abspath(self.logcat_dir), "emulator-%d.log" % self.marionette.emulator.port) if os.access(filename, os.F_OK): logfile = filename # This is all autolog stuff. # See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog from mozautolog import RESTfulAutologTestGroup testgroup = RESTfulAutologTestGroup( testgroup=self.testgroup, os="android", platform="emulator", harness="marionette", server=self.es_server, restserver=self.rest_server, machine=socket.gethostname(), logfile=logfile, ) testgroup.set_primary_product(tree=self.tree, buildtype="opt", revision=self.revision) testgroup.add_test_suite( testsuite="b2g emulator testsuite", elapsedtime=elapsedtime.seconds, cmdline="", passed=self.passed, failed=self.failed, todo=self.todo, ) # Add in the test failures. for f in self.failures: testgroup.add_test_failure(test=f[0], text=f[1], status=f[2]) testgroup.submit() def run_tests(self, tests, testtype=None): self.reset_test_stats() starttime = datetime.utcnow() while self.repeat >= 0: for test in tests: self.run_test(test, testtype) self.repeat -= 1 self.logger.info("\nSUMMARY\n-------") self.logger.info("passed: %d" % self.passed) self.logger.info("failed: %d" % self.failed) self.logger.info("todo: %d" % self.todo) try: self.marionette.check_for_crash() except: traceback.print_exc() self.elapsedtime = datetime.utcnow() - starttime if self.autolog: self.post_to_autolog(self.elapsedtime) if self.perfrequest and options.perf: try: self.perfrequest.submit() except Exception, e: print "Could not submit to datazilla" print e if self.xml_output: xml_dir = os.path.dirname(os.path.abspath(self.xml_output)) if not os.path.exists(xml_dir): os.makedirs(xml_dir) with open(self.xml_output, "w") as f: f.write(self.generate_xml(self.results)) if self.marionette.instance: self.marionette.instance.close() self.marionette.instance = None del self.marionette
class MarionetteTestRunner(object): def __init__(self, address=None, emulator=None, emulatorBinary=None, emulatorImg=None, emulator_res='480x800', homedir=None, app=None, bin=None, profile=None, autolog=False, revision=None, es_server=None, rest_server=None, logger=None, testgroup="marionette", noWindow=False, logcat_dir=None, xml_output=None, repeat=0, perf=False, perfserv=None, gecko_path=None, testvars=None, tree=None, type=None, device=None, symbols_path=None, **kwargs): self.address = address self.emulator = emulator self.emulatorBinary = emulatorBinary self.emulatorImg = emulatorImg self.emulator_res = emulator_res self.homedir = homedir self.app = app self.bin = bin self.profile = profile self.autolog = autolog self.testgroup = testgroup self.revision = revision self.es_server = es_server self.rest_server = rest_server self.logger = logger self.noWindow = noWindow self.httpd = None self.baseurl = None self.marionette = None self.logcat_dir = logcat_dir self.perfrequest = None self.xml_output = xml_output self.repeat = repeat self.perf = perf self.perfserv = perfserv self.gecko_path = gecko_path self.testvars = {} self.test_kwargs = kwargs self.tree = tree self.type = type self.device = device self.symbols_path = symbols_path if testvars: if not os.path.exists(testvars): raise Exception('--testvars file does not exist') import json with open(testvars) as f: self.testvars = json.loads(f.read()) # set up test handlers self.test_handlers = [] self.register_handlers() self.reset_test_stats() if self.logger is None: self.logger = logging.getLogger('Marionette') self.logger.setLevel(logging.INFO) self.logger.addHandler(logging.StreamHandler()) if self.logcat_dir: if not os.access(self.logcat_dir, os.F_OK): os.mkdir(self.logcat_dir) # for XML output self.testvars['xml_output'] = self.xml_output self.results = [] def reset_test_stats(self): self.passed = 0 self.failed = 0 self.todo = 0 self.failures = [] self.perfrequest = None def start_httpd(self): host = moznetwork.get_ip() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("",0)) port = s.getsockname()[1] s.close() self.baseurl = 'http://%s:%d/' % (host, port) self.logger.info('running webserver on %s' % self.baseurl) self.httpd = MozHttpd(host=host, port=port, docroot=os.path.join(os.path.dirname(__file__), 'www')) self.httpd.start() def start_marionette(self): assert(self.baseurl is not None) if self.bin: if self.address: host, port = self.address.split(':') else: host = 'localhost' port = 2828 self.marionette = Marionette(host=host, port=int(port), app=self.app, bin=self.bin, profile=self.profile, baseurl=self.baseurl) elif self.address: host, port = self.address.split(':') if self.emulator: self.marionette = Marionette.getMarionetteOrExit( host=host, port=int(port), connectToRunningEmulator=True, homedir=self.homedir, baseurl=self.baseurl, logcat_dir=self.logcat_dir, gecko_path=self.gecko_path, symbols_path=self.symbols_path) else: self.marionette = Marionette(host=host, port=int(port), baseurl=self.baseurl) elif self.emulator: self.marionette = Marionette.getMarionetteOrExit( emulator=self.emulator, emulatorBinary=self.emulatorBinary, emulatorImg=self.emulatorImg, emulator_res=self.emulator_res, homedir=self.homedir, baseurl=self.baseurl, noWindow=self.noWindow, logcat_dir=self.logcat_dir, gecko_path=self.gecko_path, symbols_path=self.symbols_path) else: raise Exception("must specify binary, address or emulator") def post_to_autolog(self, elapsedtime): self.logger.info('posting results to autolog') logfile = None if self.emulator: filename = os.path.join(os.path.abspath(self.logcat_dir), "emulator-%d.log" % self.marionette.emulator.port) if os.access(filename, os.F_OK): logfile = filename # This is all autolog stuff. # See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog from mozautolog import RESTfulAutologTestGroup testgroup = RESTfulAutologTestGroup( testgroup = self.testgroup, os = 'android', platform = 'emulator', harness = 'marionette', server = self.es_server, restserver = self.rest_server, machine = socket.gethostname(), logfile = logfile) testgroup.set_primary_product( tree = self.tree, buildtype = 'opt', revision = self.revision) testgroup.add_test_suite( testsuite = 'b2g emulator testsuite', elapsedtime = elapsedtime.seconds, cmdline = '', passed = self.passed, failed = self.failed, todo = self.todo) # Add in the test failures. for f in self.failures: testgroup.add_test_failure(test=f[0], text=f[1], status=f[2]) testgroup.submit() def run_tests(self, tests): self.reset_test_stats() starttime = datetime.utcnow() while self.repeat >=0: for test in tests: self.run_test(test) self.repeat -= 1 self.logger.info('\nSUMMARY\n-------') self.logger.info('passed: %d' % self.passed) self.logger.info('failed: %d' % self.failed) self.logger.info('todo: %d' % self.todo) try: self.marionette.check_for_crash() except: traceback.print_exc() self.elapsedtime = datetime.utcnow() - starttime if self.autolog: self.post_to_autolog(self.elapsedtime) if self.perfrequest and options.perf: try: self.perfrequest.submit() except Exception, e: print "Could not submit to datazilla" print e if self.xml_output: xml_dir = os.path.dirname(os.path.abspath(self.xml_output)) if not os.path.exists(xml_dir): os.makedirs(xml_dir) with open(self.xml_output, 'w') as f: f.write(self.generate_xml(self.results)) if self.marionette.instance: self.marionette.instance.close() self.marionette.instance = None del self.marionette