def run(arguments=sys.argv[1:]): # parse the command line arguments (options, command) = parse_args(arguments) # ensure the binary is given if not options.binary: print "Please provide a path to your Firefox binary: -b, --binary" sys.exit(1) # Parse the manifest mp = TestManifest(manifests=(options.manifest,), strict=False) # run + report if command == "testpy": results = test_all_python(mp.get(tests=mp.active_tests(disabled=False), type='python'), options) if results.failures or results.errors: sys.exit(report(True, results, None, options)) else: sys.exit(report(False)) elif command == "testjs": results = test_all_js(mp.get(tests=mp.active_tests(disabled=False), type='javascript'), options) if results.failures: sys.exit(report(True, None, results, options)) else: sys.exit(report(False)) elif command == "testall": test_all(mp.active_tests(disabled=False), options)
def run(arguments=sys.argv[1:]): # parse the command line arguments parser_kwargs = dict(arguments=arguments) (options, command) = parse_args(**parser_kwargs) # Parse the manifest mp = TestManifest(manifests=(options.manifest,)) # run + report if command == "testpy": results = test_all_python(mp.get(tests=mp.active_tests(disabled=False), type='python'), options) if results.failures or results.errors: sys.exit(report(True, results, None, options)) else: sys.exit(report(False)) elif command == "testjs": results = test_all_js(mp.get(tests=mp.active_tests(disabled=False), type='javascript'), options) if results.failures: sys.exit(report(True, None, results, options)) else: sys.exit(report(False)) elif command == "testall": test_all(mp.active_tests(disabled=False), options)
def run(arguments=sys.argv[1:]): # parse the command line arguments (options, command) = parse_args(arguments) # ensure the binary is given if not options.binary: print "Please provide a path to your Firefox binary: -b, --binary" sys.exit(1) # set the BROWSER_PATH environment variable so that # subshells will be able to invoke mozrunner os.environ['BROWSER_PATH'] = options.binary # Parse the manifest mp = TestManifest(manifests=(options.manifest,), strict=False) # run + report if command == "testpy": tests = mp.active_tests(disabled=False) results = test_all_python(mp.get(tests=tests, type='python'), options) if results.failures or results.errors: sys.exit(report(True, results, None, options)) else: sys.exit(report(False)) elif command == "testjs": tests = mp.active_tests(disabled=False) results = test_all_js(mp.get(tests=tests, type='javascript'), options) if results.fails: sys.exit(report(True, None, results, options)) else: sys.exit(report(False)) elif command == "testall": test_all(mp.active_tests(disabled=False), options)
def __init__(self, options, **kwargs): self.options = options self.server = None self.logger = mozlog.getLogger('PEP') # create the profile self.profile = self.profile_class( profile=self.options.profilePath, addons=[os.path.join(here, 'extension')]) # fork a server to serve the test related files if self.options.serverPath: self.runServer() tests = [] # TODO is there a better way of doing this? if self.options.testPath.endswith('.js'): # a single test file was passed in testObj = {} testObj['path'] = os.path.realpath(self.options.testPath) testObj['name'] = os.path.basename(self.options.testPath) testObj['here'] = os.path.dirname(testObj['path']) tests.append(testObj) else: # a test manifest was passed in # open and convert the manifest to json manifest = TestManifest() manifest.read(self.options.testPath) tests = manifest.get() # create a manifest object to be read by the JS side manifestObj = {} manifestObj['tests'] = tests # write manifest to a JSON file jsonManifest = open(os.path.join(here, 'manifest.json'), 'w') jsonManifest.write(str(manifestObj).replace("'", "\"")) jsonManifest.close() # setup environment env = os.environ.copy() env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1' env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = str( options.tracerThreshold) env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = str(options.tracerInterval) env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' # construct the browser arguments cmdargs = [] # TODO Make browserArgs a list cmdargs.extend(self.options.browserArgs) cmdargs.extend(['-pep-start', os.path.realpath(jsonManifest.name)]) # run with managed process handler self.runner = self.runner_class(profile=self.profile, binary=self.options.binary, cmdargs=cmdargs, env=env, process_class=PepProcess)
def __init__(self, options, **kwargs): self.options = options self.server = None self.logger = mozlog.getLogger('PEP') # create the profile self.profile = self.profile_class(profile=self.options.profilePath, addons=[os.path.join(here, 'extension')]) # fork a server to serve the test related files if self.options.serverPath: self.runServer() tests = [] # TODO is there a better way of doing this? if self.options.testPath.endswith('.js'): # a single test file was passed in testObj = {} testObj['path'] = os.path.realpath(self.options.testPath) testObj['name'] = os.path.basename(self.options.testPath) tests.append(testObj) else: # a test manifest was passed in # open and convert the manifest to json manifest = TestManifest() manifest.read(self.options.testPath) tests = manifest.get() # create a manifest object to be read by the JS side manifestObj = {} manifestObj['tests'] = tests # write manifest to a JSON file jsonManifest = open(os.path.join(here, 'manifest.json'), 'w') jsonManifest.write(str(manifestObj).replace("'", "\"")) jsonManifest.close() # setup environment env = os.environ.copy() env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1' env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = str(options.tracerThreshold) env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = str(options.tracerInterval) env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' # construct the browser arguments cmdargs = [] # TODO Make browserArgs a list cmdargs.extend(self.options.browserArgs) cmdargs.extend(['-pep-start', os.path.realpath(jsonManifest.name)]) # run with managed process handler self.runner = self.runner_class(profile=self.profile, binary=self.options.binary, cmdargs=cmdargs, env=env, process_class=PepProcess)
def get_test_list(self, manifest): self.logger.info("Reading test manifest: %s" % manifest) mft = TestManifest() mft.read(manifest) # In the future if we want to add in more processing to the manifest # here is where you'd do that. Right now, we just return a list of # tests testlist = [] for i in mft.get(): testlist.append(i["path"]) return testlist
def read_tests(self): self._tests = [] manifest = TestManifest() manifest.read(self._test_path) tests_info = manifest.get() for t in tests_info: if not t['here'] in sys.path: sys.path.append(t['here']) if t['name'].endswith('.py'): t['name'] = t['name'][:-3] # add all classes in module that are derived from PhoneTest to # the test list tests = [(x[1], os.path.normpath(os.path.join(t['here'], t.get('config', '')))) for x in inspect.getmembers(__import__(t['name']), inspect.isclass) if x[0] != 'PhoneTest' and issubclass(x[1], phonetest.PhoneTest)] self._tests.extend(tests)
def print_test_dirs(topsrcdir, manifest_file): """ Simple routine which prints the paths of directories specified in a Marionette manifest, relative to topsrcdir. This does not recurse into manifests, as we currently have no need for that. """ # output the directory of this (parent) manifest topsrcdir = os.path.abspath(topsrcdir) scriptdir = os.path.abspath(os.path.dirname(__file__)) print scriptdir[len(topsrcdir) + 1:] # output the directories of all the other manifests dirs = set() manifest = TestManifest() manifest.read(manifest_file) for i in manifest.get(): dirs.add(os.path.dirname(i['manifest'])[len(topsrcdir) + 1:]) for path in dirs: print path
def print_test_dirs(topsrcdir, manifest_file): """ Simple routine which prints the paths of directories specified in a Marionette manifest, relative to topsrcdir. This does not recurse into manifests, as we currently have no need for that. """ dirs = set() # output the directory of this (parent) manifest topsrcdir = os.path.abspath(topsrcdir) scriptdir = os.path.abspath(os.path.dirname(__file__)) dirs.add(scriptdir[len(topsrcdir) + 1:]) # output the directories of all the other manifests manifest = TestManifest() manifest.read(manifest_file) for i in manifest.get(): d = os.path.dirname(i['manifest'])[len(topsrcdir) + 1:] dirs.add(d) for path in dirs: path = path.replace('\\', '/') print path
def add_test(self, test, expected='pass', test_container=None): filepath = os.path.abspath(test) if os.path.isdir(filepath): for root, dirs, files in os.walk(filepath): for filename in files: if (filename.startswith('test_') and (filename.endswith('.py') or filename.endswith('.js'))): filepath = os.path.join(root, filename) self.add_test(filepath) return testargs = {} if self.type is not None: testtypes = self.type.replace('+', ' +').replace('-', ' -').split() for atype in testtypes: if atype.startswith('+'): testargs.update({ atype[1:]: 'true' }) elif atype.startswith('-'): testargs.update({ atype[1:]: 'false' }) else: testargs.update({ atype: 'true' }) testarg_b2g = bool(testargs.get('b2g')) file_ext = os.path.splitext(os.path.split(filepath)[-1])[1] if file_ext == '.ini': manifest = TestManifest() manifest.read(filepath) filters = [] if self.test_tags: filters.append(tags(self.test_tags)) manifest_tests = manifest.active_tests(exists=False, disabled=True, filters=filters, device=self.device, app=self.appName, **mozinfo.info) if len(manifest_tests) == 0: self.logger.error("no tests to run using specified " "combination of filters: {}".format( manifest.fmt_filters())) unfiltered_tests = [] for test in manifest_tests: if test.get('disabled'): self.manifest_skipped_tests.append(test) else: unfiltered_tests.append(test) target_tests = manifest.get(tests=unfiltered_tests, **testargs) for test in unfiltered_tests: if test['path'] not in [x['path'] for x in target_tests]: test.setdefault('disabled', 'filtered by type (%s)' % self.type) self.manifest_skipped_tests.append(test) for i in target_tests: if not os.path.exists(i["path"]): raise IOError("test file: %s does not exist" % i["path"]) file_ext = os.path.splitext(os.path.split(i['path'])[-1])[-1] test_container = None if i.get('test_container') and testarg_b2g: if i.get('test_container') == "true": test_container = True elif i.get('test_container') == "false": test_container = False self.add_test(i["path"], i["expected"], test_container) return self.tests.append({'filepath': filepath, 'expected': expected, 'test_container': test_container})
def run_test(self, test, expected='pass'): if not self.httpd: print "starting httpd" self.start_httpd() if not self.marionette: self.start_marionette() if self.emulator: self.marionette.emulator.wait_for_homescreen(self.marionette) testargs = {} if self.type is not None: testtypes = self.type.replace('+', ' +').replace('-', ' -').split() for atype in testtypes: if atype.startswith('+'): testargs.update({atype[1:]: 'true'}) elif atype.startswith('-'): testargs.update({atype[1:]: 'false'}) else: testargs.update({atype: 'true'}) oop = testargs.get('oop', False) if isinstance(oop, basestring): oop = False if oop == 'false' else 'true' filepath = os.path.abspath(test) if os.path.isdir(filepath): for root, dirs, files in os.walk(filepath): if self.shuffle: random.shuffle(files) for filename in files: if ((filename.startswith('test_') or filename.startswith('browser_')) and (filename.endswith('.py') or filename.endswith('.js'))): filepath = os.path.join(root, filename) self.run_test(filepath) if self.marionette.check_for_crash(): return return mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1]) testloader = unittest.TestLoader() suite = unittest.TestSuite() if file_ext == '.ini': manifest = TestManifest() manifest.read(filepath) all_tests = manifest.active_tests(exists=False, disabled=False) manifest_tests = manifest.active_tests(exists=False, disabled=False, device=self.device, app=self.appName) skip_tests = list( set([x['path'] for x in all_tests]) - set([x['path'] for x in manifest_tests])) for skipped in skip_tests: self.logger.info( 'TEST-SKIP | %s | device=%s, app=%s' % (os.path.basename(skipped), self.device, self.appName)) self.todo += 1 target_tests = manifest.get(tests=manifest_tests, **testargs) if self.shuffle: random.shuffle(target_tests) for i in target_tests: self.run_test(i["path"], i["expected"]) if self.marionette.check_for_crash(): return return self.logger.info('TEST-START %s' % os.path.basename(test)) self.test_kwargs['expected'] = expected self.test_kwargs['oop'] = oop for handler in self.test_handlers: if handler.match(os.path.basename(test)): handler.add_tests_to_suite(mod_name, filepath, suite, testloader, self.marionette, self.testvars, **self.test_kwargs) break if suite.countTestCases(): runner = self.textrunnerclass(verbosity=3, marionette=self.marionette) results = runner.run(suite) self.results.append(results) self.failed += len(results.failures) + len(results.errors) if hasattr(results, 'skipped'): self.todo += len(results.skipped) self.passed += results.passed for failure in results.failures + results.errors: self.failures.append((results.getInfo(failure), failure.output, 'TEST-UNEXPECTED-FAIL')) if hasattr(results, 'unexpectedSuccesses'): self.failed += len(results.unexpectedSuccesses) for failure in results.unexpectedSuccesses: self.failures.append( (results.getInfo(failure), 'TEST-UNEXPECTED-PASS')) if hasattr(results, 'expectedFailures'): self.passed += len(results.expectedFailures)
def run_test(self, test, testtype): if not self.httpd: self.start_httpd() if not self.marionette: self.start_marionette() filepath = os.path.abspath(test) if os.path.isdir(filepath): for root, dirs, files in os.walk(filepath): for filename in files: if ((filename.startswith('test_') or filename.startswith('browser_')) and (filename.endswith('.py') or filename.endswith('.js'))): filepath = os.path.join(root, filename) self.run_test(filepath, testtype) return mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1]) testloader = unittest.TestLoader() suite = unittest.TestSuite() if file_ext == '.ini': if testtype is not None: testargs = {} testtypes = testtype.replace('+', ' +').replace('-', ' -').split() for atype in testtypes: if atype.startswith('+'): testargs.update({ atype[1:]: 'true' }) elif atype.startswith('-'): testargs.update({ atype[1:]: 'false' }) else: testargs.update({ atype: 'true' }) manifest = TestManifest() manifest.read(filepath) if testtype is None: manifest_tests = manifest.get() else: manifest_tests = manifest.get(**testargs) for i in manifest_tests: self.run_test(i["path"], testtype) return self.logger.info('TEST-START %s' % os.path.basename(test)) if file_ext == '.py': test_mod = imp.load_source(mod_name, filepath) for name in dir(test_mod): obj = getattr(test_mod, name) if (isinstance(obj, (type, types.ClassType)) and issubclass(obj, unittest.TestCase)): testnames = testloader.getTestCaseNames(obj) for testname in testnames: suite.addTest(obj(self.marionette, methodName=testname)) elif file_ext == '.js': suite.addTest(MarionetteJSTestCase(self.marionette, jsFile=filepath)) if suite.countTestCases(): results = MarionetteTextTestRunner(verbosity=3).run(suite) self.failed += len(results.failures) + len(results.errors) self.todo = 0 if hasattr(results, 'skipped'): self.todo += len(results.skipped) + len(results.expectedFailures) self.passed += results.passed for failure in results.failures + results.errors: self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL')) if hasattr(results, 'unexpectedSuccess'): self.failed += len(results.unexpectedSuccesses) for failure in results.unexpectedSuccesses: self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-PASS'))
def __init__(self, options, **kwargs): self.options = options self.server = None self.logger = mozlog.getLogger('PEP') # create the profile enable_proxy = False locations = ServerLocations() if self.options.proxyLocations: if not self.options.serverPath: self.logger.warning('Can\'t set up proxy without server path') else: enable_proxy = True for proxyLocation in self.options.proxyLocations: locations.read(proxyLocation, False) locations.add_host(host='127.0.0.1', port=self.options.serverPort, options='primary,privileged') self.profile = self.profile_class( profile=self.options.profilePath, addons=[os.path.join(here, 'extension')], locations=locations, proxy=enable_proxy) # fork a server to serve the test related files if self.options.serverPath: self.runServer() tests = [] # TODO is there a better way of doing this? if self.options.testPath.endswith('.js'): # a single test file was passed in testObj = {} testObj['path'] = os.path.realpath(self.options.testPath) testObj['name'] = os.path.basename(self.options.testPath) testObj['here'] = os.path.dirname(testObj['path']) tests.append(testObj) else: # a test manifest was passed in # open and convert the manifest to json manifest = TestManifest() manifest.read(self.options.testPath) tests = manifest.get() # create a manifest object to be read by the JS side manifestObj = {} manifestObj['tests'] = tests manifestObj['options'] = options.__dict__ # write manifest to a JSON file jsonManifest = open(os.path.join(here, 'manifest.json'), 'w') jsonManifest.write(json.dumps(manifestObj)) jsonManifest.close() # setup environment env = os.environ.copy() env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1' env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = str( options.tracerThreshold) env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = str(options.tracerInterval) env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' # construct the browser arguments cmdargs = [] # TODO Make browserArgs a list cmdargs.extend(self.options.browserArgs) cmdargs.extend(['-pep-start', os.path.realpath(jsonManifest.name)]) # run with managed process handler self.runner = self.runner_class(profile=self.profile, binary=self.options.binary, cmdargs=cmdargs, env=env, process_class=PepProcess)
def run_test(self, test, testtype): if not self.httpd: print "starting httpd" self.start_httpd() if not self.marionette: self.start_marionette() filepath = os.path.abspath(test) if os.path.isdir(filepath): for root, dirs, files in os.walk(filepath): for filename in files: if ((filename.startswith('test_') or filename.startswith('browser_')) and (filename.endswith('.py') or filename.endswith('.js'))): filepath = os.path.join(root, filename) self.run_test(filepath, testtype) return mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1]) testloader = unittest.TestLoader() suite = unittest.TestSuite() if file_ext == '.ini': testargs = { 'skip': 'false' } if testtype is not None: testtypes = testtype.replace('+', ' +').replace('-', ' -').split() for atype in testtypes: if atype.startswith('+'): testargs.update({ atype[1:]: 'true' }) elif atype.startswith('-'): testargs.update({ atype[1:]: 'false' }) else: testargs.update({ atype: 'true' }) manifest = TestManifest() manifest.read(filepath) if self.perf: if self.perfserv is None: self.perfserv = manifest.get("perfserv")[0] machine_name = socket.gethostname() try: manifest.has_key("machine_name") machine_name = manifest.get("machine_name")[0] except: self.logger.info("Using machine_name: %s" % machine_name) os_name = platform.system() os_version = platform.release() self.perfrequest = datazilla.DatazillaRequest( server=self.perfserv, machine_name=machine_name, os=os_name, os_version=os_version, platform=manifest.get("platform")[0], build_name=manifest.get("build_name")[0], version=manifest.get("version")[0], revision=self.revision, branch=manifest.get("branch")[0], id=os.getenv('BUILD_ID'), test_date=int(time.time())) manifest_tests = manifest.get(**testargs) for i in manifest_tests: self.run_test(i["path"], testtype) return self.logger.info('TEST-START %s' % os.path.basename(test)) for handler in self.test_handlers: if handler.match(os.path.basename(test)): handler.add_tests_to_suite(mod_name, filepath, suite, testloader, self.marionette) break if suite.countTestCases(): results = MarionetteTextTestRunner(verbosity=3, perf=self.perf).run(suite) self.results.append(results) self.failed += len(results.failures) + len(results.errors) if results.perfdata and options.perf: self.perfrequest.add_datazilla_result(results.perfdata) if hasattr(results, 'skipped'): self.todo += len(results.skipped) + len(results.expectedFailures) self.passed += results.passed for failure in results.failures + results.errors: self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL')) if hasattr(results, 'unexpectedSuccess'): self.failed += len(results.unexpectedSuccesses) for failure in results.unexpectedSuccesses: self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-PASS'))
def run_test(self, test): if not self.httpd: print "starting httpd" self.start_httpd() if not self.marionette: self.start_marionette() filepath = os.path.abspath(test) if os.path.isdir(filepath): for root, dirs, files in os.walk(filepath): for filename in files: if ((filename.startswith('test_') or filename.startswith('browser_')) and (filename.endswith('.py') or filename.endswith('.js'))): filepath = os.path.join(root, filename) self.run_test(filepath) if self.marionette.check_for_crash(): return return mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1]) testloader = unittest.TestLoader() suite = unittest.TestSuite() if file_ext == '.ini': testargs = {} if self.type is not None: testtypes = self.type.replace('+', ' +').replace('-', ' -').split() for atype in testtypes: if atype.startswith('+'): testargs.update({ atype[1:]: 'true' }) elif atype.startswith('-'): testargs.update({ atype[1:]: 'false' }) else: testargs.update({ atype: 'true' }) manifest = TestManifest() manifest.read(filepath) all_tests = manifest.active_tests(disabled=False) manifest_tests = manifest.active_tests(disabled=False, device=self.device, app=self.appName) skip_tests = list(set([x['path'] for x in all_tests]) - set([x['path'] for x in manifest_tests])) for skipped in skip_tests: self.logger.info('TEST-SKIP | %s | device=%s, app=%s' % (os.path.basename(skipped), self.device, self.appName)) self.todo += 1 for i in manifest.get(tests=manifest_tests, **testargs): self.run_test(i["path"]) if self.marionette.check_for_crash(): return return self.logger.info('TEST-START %s' % os.path.basename(test)) for handler in self.test_handlers: if handler.match(os.path.basename(test)): handler.add_tests_to_suite(mod_name, filepath, suite, testloader, self.marionette, self.testvars, **self.test_kwargs) break if suite.countTestCases(): runner = self.textrunnerclass(verbosity=3, marionette=self.marionette) results = runner.run(suite) self.results.append(results) self.failed += len(results.failures) + len(results.errors) if hasattr(results, 'skipped'): self.todo += len(results.skipped) self.passed += results.passed for failure in results.failures + results.errors: self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL')) if hasattr(results, 'unexpectedSuccesses'): self.failed += len(results.unexpectedSuccesses) for failure in results.unexpectedSuccesses: self.failures.append((results.getInfo(failure), 'TEST-UNEXPECTED-PASS')) if hasattr(results, 'expectedFailures'): self.passed += len(results.expectedFailures)
def run_test(self, test, testtype): if not self.httpd: print "starting httpd" self.start_httpd() if not self.marionette: self.start_marionette() filepath = os.path.abspath(test) if os.path.isdir(filepath): for root, dirs, files in os.walk(filepath): for filename in files: if ((filename.startswith('test_') or filename.startswith('browser_')) and (filename.endswith('.py') or filename.endswith('.js'))): filepath = os.path.join(root, filename) self.run_test(filepath, testtype) if self.marionette.check_for_crash(): return return mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1]) testloader = unittest.TestLoader() suite = unittest.TestSuite() if file_ext == '.ini': testargs = {} if testtype is not None: testtypes = testtype.replace('+', ' +').replace('-', ' -').split() for atype in testtypes: if atype.startswith('+'): testargs.update({ atype[1:]: 'true' }) elif atype.startswith('-'): testargs.update({ atype[1:]: 'false' }) else: testargs.update({ atype: 'true' }) manifest = TestManifest() manifest.read(filepath) if self.perf: if self.perfserv is None: self.perfserv = manifest.get("perfserv")[0] machine_name = socket.gethostname() try: manifest.has_key("machine_name") machine_name = manifest.get("machine_name")[0] except: self.logger.info("Using machine_name: %s" % machine_name) os_name = platform.system() os_version = platform.release() self.perfrequest = datazilla.DatazillaRequest( server=self.perfserv, machine_name=machine_name, os=os_name, os_version=os_version, platform=manifest.get("platform")[0], build_name=manifest.get("build_name")[0], version=manifest.get("version")[0], revision=self.revision, branch=manifest.get("branch")[0], id=os.getenv('BUILD_ID'), test_date=int(time.time())) manifest_tests = manifest.active_tests(disabled=False) for i in manifest.get(tests=manifest_tests, **testargs): self.run_test(i["path"], testtype) if self.marionette.check_for_crash(): return return self.logger.info('TEST-START %s' % os.path.basename(test)) for handler in self.test_handlers: if handler.match(os.path.basename(test)): handler.add_tests_to_suite(mod_name, filepath, suite, testloader, self.marionette, self.testvars) break if suite.countTestCases(): runner = MarionetteTextTestRunner(verbosity=3, perf=self.perf, marionette=self.marionette) results = runner.run(suite) self.results.append(results) self.failed += len(results.failures) + len(results.errors) if results.perfdata and options.perf: self.perfrequest.add_datazilla_result(results.perfdata) if hasattr(results, 'skipped'): self.todo += len(results.skipped) + len(results.expectedFailures) self.passed += results.passed for failure in results.failures + results.errors: self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL')) if hasattr(results, 'unexpectedSuccess'): self.failed += len(results.unexpectedSuccesses) for failure in results.unexpectedSuccesses: self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-PASS'))
def __init__(self, options, **kwargs): self.options = options self.server = None self.logger = mozlog.getLogger("PEP") # create the profile enable_proxy = False locations = ServerLocations() if self.options.proxyLocations: if not self.options.serverPath: self.logger.warning("Can't set up proxy without server path") else: enable_proxy = True locations.read(self.options.proxyLocations, False) locations.add_host(host="127.0.0.1", port=self.options.serverPort, options="primary,privileged") self.profile = self.profile_class( profile=self.options.profilePath, addons=[os.path.join(here, "extension")], locations=locations, proxy=enable_proxy, ) # fork a server to serve the test related files if self.options.serverPath: self.runServer() tests = [] # TODO is there a better way of doing this? if self.options.testPath.endswith(".js"): # a single test file was passed in testObj = {} testObj["path"] = os.path.realpath(self.options.testPath) testObj["name"] = os.path.basename(self.options.testPath) testObj["here"] = os.path.dirname(testObj["path"]) tests.append(testObj) else: # a test manifest was passed in # open and convert the manifest to json manifest = TestManifest() manifest.read(self.options.testPath) tests = manifest.get() # create a manifest object to be read by the JS side manifestObj = {} manifestObj["tests"] = tests manifestObj["options"] = options.__dict__ # write manifest to a JSON file jsonManifest = open(os.path.join(here, "manifest.json"), "w") jsonManifest.write(json.dumps(manifestObj)) jsonManifest.close() # setup environment env = os.environ.copy() env["MOZ_INSTRUMENT_EVENT_LOOP"] = "1" env["MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD"] = str(options.tracerThreshold) env["MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL"] = str(options.tracerInterval) env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" # construct the browser arguments cmdargs = [] # TODO Make browserArgs a list cmdargs.extend(self.options.browserArgs) cmdargs.extend(["-pep-start", os.path.realpath(jsonManifest.name)]) # run with managed process handler self.runner = self.runner_class( profile=self.profile, binary=self.options.binary, cmdargs=cmdargs, env=env, process_class=PepProcess )
def __init__(self, options, **kwargs): self.options = options self.server = None self.logger = mozlog.getLogger('PEP') # create the profile enable_proxy = False locations = ServerLocations() if self.options.proxyLocations: if not self.options.serverPath: self.logger.warning('Can\'t set up proxy without server path') else: enable_proxy = True for proxyLocation in self.options.proxyLocations: locations.read(proxyLocation, False) locations.add_host(host='127.0.0.1', port=self.options.serverPort, options='primary,privileged') self.profile = self.profile_class(profile=self.options.profilePath, addons=[os.path.join(here, 'extension')], locations=locations, proxy=enable_proxy) # fork a server to serve the test related files if self.options.serverPath: self.runServer() tests = [] # TODO is there a better way of doing this? if self.options.testPath.endswith('.js'): # a single test file was passed in testObj = {} testObj['path'] = os.path.realpath(self.options.testPath) testObj['name'] = os.path.basename(self.options.testPath) testObj['here'] = os.path.dirname(testObj['path']) tests.append(testObj) else: # a test manifest was passed in # open and convert the manifest to json manifest = TestManifest() manifest.read(self.options.testPath) tests = manifest.get() # create a manifest object to be read by the JS side manifestObj = {} manifestObj['tests'] = tests manifestObj['options'] = options.__dict__ # write manifest to a JSON file jsonManifest = open(os.path.join(here, 'manifest.json'), 'w') jsonManifest.write(json.dumps(manifestObj)) jsonManifest.close() # setup environment env = os.environ.copy() env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1' env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = str(options.tracerThreshold) env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = str(options.tracerInterval) env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' # construct the browser arguments cmdargs = [] # TODO Make browserArgs a list cmdargs.extend(self.options.browserArgs) cmdargs.extend(['-pep-start', os.path.realpath(jsonManifest.name)]) # run with managed process handler self.runner = self.runner_class(profile=self.profile, binary=self.options.binary, cmdargs=cmdargs, env=env, process_class=PepProcess)
def add_test(self, test, expected='pass', test_container=None): filepath = os.path.abspath(test) if os.path.isdir(filepath): for root, dirs, files in os.walk(filepath): for filename in files: if (filename.startswith('test_') and (filename.endswith('.py') or filename.endswith('.js'))): filepath = os.path.join(root, filename) self.add_test(filepath) return testargs = {} if self.type is not None: testtypes = self.type.replace('+', ' +').replace('-', ' -').split() for atype in testtypes: if atype.startswith('+'): testargs.update({ atype[1:]: 'true' }) elif atype.startswith('-'): testargs.update({ atype[1:]: 'false' }) else: testargs.update({ atype: 'true' }) testarg_b2g = bool(testargs.get('b2g')) file_ext = os.path.splitext(os.path.split(filepath)[-1])[1] if file_ext == '.ini': manifest = TestManifest() manifest.read(filepath) filters = [] if self.test_tags: filters.append(tags(self.test_tags)) json_path = update_mozinfo(filepath) self.logger.info("mozinfo updated with the following: {}".format(None)) manifest_tests = manifest.active_tests(exists=False, disabled=True, filters=filters, device=self.device, app=self.appName, e10s=self.e10s, **mozinfo.info) if len(manifest_tests) == 0: self.logger.error("no tests to run using specified " "combination of filters: {}".format( manifest.fmt_filters())) unfiltered_tests = [] for test in manifest_tests: if test.get('disabled'): self.manifest_skipped_tests.append(test) else: unfiltered_tests.append(test) target_tests = manifest.get(tests=unfiltered_tests, **testargs) for test in unfiltered_tests: if test['path'] not in [x['path'] for x in target_tests]: test.setdefault('disabled', 'filtered by type (%s)' % self.type) self.manifest_skipped_tests.append(test) for i in target_tests: if not os.path.exists(i["path"]): raise IOError("test file: %s does not exist" % i["path"]) file_ext = os.path.splitext(os.path.split(i['path'])[-1])[-1] test_container = None if i.get('test_container') and testarg_b2g: if i.get('test_container') == "true": test_container = True elif i.get('test_container') == "false": test_container = False self.add_test(i["path"], i["expected"], test_container) return self.tests.append({'filepath': filepath, 'expected': expected, 'test_container': test_container})
def run_test(self, test, testtype): if not self.httpd: self.start_httpd() if not self.marionette: self.start_marionette() if not os.path.isabs(test): filepath = os.path.join(os.path.dirname(__file__), test) else: filepath = test if os.path.isdir(filepath): for root, dirs, files in os.walk(filepath): for filename in files: if ((filename.startswith('test_') or filename.startswith('browser_')) and (filename.endswith('.py') or filename.endswith('.js'))): filepath = os.path.join(root, filename) self.run_test(filepath, testtype) return mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1]) testloader = unittest.TestLoader() suite = unittest.TestSuite() if file_ext == '.ini': if testtype is not None: testargs = {} testtypes = testtype.replace('+', ' +').replace('-', ' -').split() for atype in testtypes: if atype.startswith('+'): testargs.update({atype[1:]: 'true'}) elif atype.startswith('-'): testargs.update({atype[1:]: 'false'}) else: testargs.update({atype: 'true'}) manifest = TestManifest() manifest.read(filepath) if testtype is None: manifest_tests = manifest.get() else: manifest_tests = manifest.get(**testargs) for i in manifest_tests: self.run_test(i["path"], testtype) return self.logger.info('TEST-START %s' % os.path.basename(test)) if file_ext == '.py': test_mod = imp.load_source(mod_name, filepath) for name in dir(test_mod): obj = getattr(test_mod, name) if (isinstance(obj, (type, types.ClassType)) and issubclass(obj, unittest.TestCase)): testnames = testloader.getTestCaseNames(obj) for testname in testnames: suite.addTest(obj(self.marionette, methodName=testname)) elif file_ext == '.js': suite.addTest( MarionetteJSTestCase(self.marionette, jsFile=filepath)) if suite.countTestCases(): results = MarionetteTextTestRunner(verbosity=3).run(suite) self.failed += len(results.failures) + len(results.errors) self.todo = 0 if hasattr(results, 'skipped'): self.todo += len(results.skipped) + len( results.expectedFailures) self.passed += results.passed for failure in results.failures + results.errors: self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL')) if hasattr(results, 'unexpectedSuccess'): self.failed += len(results.unexpectedSuccesses) for failure in results.unexpectedSuccesses: self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-PASS'))
def run_test(self, test): if not self.httpd: print "starting httpd" self.start_httpd() if not self.marionette: self.start_marionette() filepath = os.path.abspath(test) if os.path.isdir(filepath): for root, dirs, files in os.walk(filepath): for filename in files: if ((filename.startswith('test_') or filename.startswith('browser_')) and (filename.endswith('.py') or filename.endswith('.js'))): filepath = os.path.join(root, filename) self.run_test(filepath) if self.marionette.check_for_crash(): return return mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1]) testloader = unittest.TestLoader() suite = unittest.TestSuite() if file_ext == '.ini': testargs = {} if self.type is not None: testtypes = self.type.replace('+', ' +').replace('-', ' -').split() for atype in testtypes: if atype.startswith('+'): testargs.update({atype[1:]: 'true'}) elif atype.startswith('-'): testargs.update({atype[1:]: 'false'}) else: testargs.update({atype: 'true'}) manifest = TestManifest() manifest.read(filepath) manifest_tests = manifest.active_tests(disabled=False) for i in manifest.get(tests=manifest_tests, **testargs): self.run_test(i["path"]) if self.marionette.check_for_crash(): return return self.logger.info('TEST-START %s' % os.path.basename(test)) for handler in self.test_handlers: if handler.match(os.path.basename(test)): handler.add_tests_to_suite(mod_name, filepath, suite, testloader, self.marionette, self.testvars, **self.test_kwargs) break if suite.countTestCases(): runner = MarionetteTextTestRunner(verbosity=3, marionette=self.marionette) results = runner.run(suite) self.results.append(results) self.failed += len(results.failures) + len(results.errors) if hasattr(results, 'skipped'): self.todo += len(results.skipped) + len( results.expectedFailures) self.passed += results.passed for failure in results.failures + results.errors: self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL')) if hasattr(results, 'unexpectedSuccess'): self.failed += len(results.unexpectedSuccesses) for failure in results.unexpectedSuccesses: self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-PASS'))
def add_test(self, test, expected="pass", test_container=None): filepath = os.path.abspath(test) if os.path.isdir(filepath): for root, dirs, files in os.walk(filepath): for filename in files: if filename.startswith("test_") and (filename.endswith(".py") or filename.endswith(".js")): filepath = os.path.join(root, filename) self.add_test(filepath) return testargs = {} if self.type is not None: testtypes = self.type.replace("+", " +").replace("-", " -").split() for atype in testtypes: if atype.startswith("+"): testargs.update({atype[1:]: "true"}) elif atype.startswith("-"): testargs.update({atype[1:]: "false"}) else: testargs.update({atype: "true"}) testarg_b2g = bool(testargs.get("b2g")) file_ext = os.path.splitext(os.path.split(filepath)[-1])[1] if file_ext == ".ini": manifest = TestManifest() manifest.read(filepath) filters = [] if self.test_tags: filters.append(tags(self.test_tags)) manifest_tests = manifest.active_tests( exists=False, disabled=True, filters=filters, device=self.device, app=self.appName, **mozinfo.info ) if len(manifest_tests) == 0: self.logger.error( "no tests to run using specified " "combination of filters: {}".format(manifest.fmt_filters()) ) unfiltered_tests = [] for test in manifest_tests: if test.get("disabled"): self.manifest_skipped_tests.append(test) else: unfiltered_tests.append(test) target_tests = manifest.get(tests=unfiltered_tests, **testargs) for test in unfiltered_tests: if test["path"] not in [x["path"] for x in target_tests]: test.setdefault("disabled", "filtered by type (%s)" % self.type) self.manifest_skipped_tests.append(test) for i in target_tests: if not os.path.exists(i["path"]): raise IOError("test file: %s does not exist" % i["path"]) file_ext = os.path.splitext(os.path.split(i["path"])[-1])[-1] test_container = None if i.get("test_container") and testarg_b2g: if i.get("test_container") == "true": test_container = True elif i.get("test_container") == "false": test_container = False self.add_test(i["path"], i["expected"], test_container) return self.tests.append({"filepath": filepath, "expected": expected, "test_container": test_container})
def run_test(self, test, testtype): if not self.httpd: print "starting httpd" self.start_httpd() if not self.marionette: self.start_marionette() filepath = os.path.abspath(test) if os.path.isdir(filepath): for root, dirs, files in os.walk(filepath): for filename in files: if (filename.startswith("test_") or filename.startswith("browser_")) and ( filename.endswith(".py") or filename.endswith(".js") ): filepath = os.path.join(root, filename) self.run_test(filepath, testtype) return mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1]) testloader = unittest.TestLoader() suite = unittest.TestSuite() if file_ext == ".ini": testargs = {"skip": "false"} if testtype is not None: testtypes = testtype.replace("+", " +").replace("-", " -").split() for atype in testtypes: if atype.startswith("+"): testargs.update({atype[1:]: "true"}) elif atype.startswith("-"): testargs.update({atype[1:]: "false"}) else: testargs.update({atype: "true"}) manifest = TestManifest() manifest.read(filepath) if options.perf: if options.perfserv is None: options.perfserv = manifest.get("perfserv")[0] machine_name = socket.gethostname() try: manifest.has_key("machine_name") machine_name = manifest.get("machine_name")[0] except: self.logger.info("Using machine_name: %s" % machine_name) os_name = platform.system() os_version = platform.release() self.perfrequest = datazilla.DatazillaRequest( server=options.perfserv, machine_name=machine_name, os=os_name, os_version=os_version, platform=manifest.get("platform")[0], build_name=manifest.get("build_name")[0], version=manifest.get("version")[0], revision=self.revision, branch=manifest.get("branch")[0], id=os.getenv("BUILD_ID"), test_date=int(time.time()), ) manifest_tests = manifest.get(**testargs) for i in manifest_tests: self.run_test(i["path"], testtype) return self.logger.info("TEST-START %s" % os.path.basename(test)) if file_ext == ".py": test_mod = imp.load_source(mod_name, filepath) for name in dir(test_mod): obj = getattr(test_mod, name) if isinstance(obj, (type, types.ClassType)) and issubclass(obj, unittest.TestCase): testnames = testloader.getTestCaseNames(obj) for testname in testnames: suite.addTest(obj(self.marionette, methodName=testname)) elif file_ext == ".js": suite.addTest(MarionetteJSTestCase(self.marionette, jsFile=filepath)) if suite.countTestCases(): results = MarionetteTextTestRunner(verbosity=3).run(suite) self.results.append(results) self.failed += len(results.failures) + len(results.errors) if results.perfdata and options.perf: self.perfrequest.add_datazilla_result(results.perfdata) if hasattr(results, "skipped"): self.todo += len(results.skipped) + len(results.expectedFailures) self.passed += results.passed for failure in results.failures + results.errors: self.failures.append((results.getInfo(failure[0]), failure[1], "TEST-UNEXPECTED-FAIL")) if hasattr(results, "unexpectedSuccess"): self.failed += len(results.unexpectedSuccesses) for failure in results.unexpectedSuccesses: self.failures.append((results.getInfo(failure[0]), failure[1], "TEST-UNEXPECTED-PASS"))
def run_test(self, test, expected='pass'): if not self.httpd: print "starting httpd" self.start_httpd() if not self.marionette: self.start_marionette() if self.emulator: self.marionette.emulator.wait_for_homescreen(self.marionette) # Retrieve capabilities for later use if not self._capabilities: self.capabilities testargs = {} if self.type is not None: testtypes = self.type.replace('+', ' +').replace('-', ' -').split() for atype in testtypes: if atype.startswith('+'): testargs.update({ atype[1:]: 'true' }) elif atype.startswith('-'): testargs.update({ atype[1:]: 'false' }) else: testargs.update({ atype: 'true' }) oop = testargs.get('oop', False) if isinstance(oop, basestring): oop = False if oop == 'false' else 'true' filepath = os.path.abspath(test) if os.path.isdir(filepath): for root, dirs, files in os.walk(filepath): if self.shuffle: random.shuffle(files) for filename in files: if ((filename.startswith('test_') or filename.startswith('browser_')) and (filename.endswith('.py') or filename.endswith('.js'))): filepath = os.path.join(root, filename) self.run_test(filepath) if self.marionette.check_for_crash(): return return mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1]) testloader = unittest.TestLoader() suite = unittest.TestSuite() if file_ext == '.ini': manifest = TestManifest() manifest.read(filepath) manifest_tests = manifest.active_tests(exists=False, disabled=True, device=self.device, app=self.appName, **mozinfo.info) unfiltered_tests = [] for test in manifest_tests: if test.get('disabled'): self.manifest_skipped_tests.append(test) else: unfiltered_tests.append(test) target_tests = manifest.get(tests=unfiltered_tests, **testargs) for test in unfiltered_tests: if test['path'] not in [x['path'] for x in target_tests]: test.setdefault('disabled', 'filtered by type (%s)' % self.type) self.manifest_skipped_tests.append(test) for test in self.manifest_skipped_tests: self.logger.info('TEST-SKIP | %s | %s' % ( os.path.basename(test['path']), test['disabled'])) self.todo += 1 if self.shuffle: random.shuffle(target_tests) for i in target_tests: if not os.path.exists(i["path"]): raise IOError("test file: %s does not exist" % i["path"]) self.run_test(i["path"], i["expected"]) if self.marionette.check_for_crash(): return return self.logger.info('TEST-START %s' % os.path.basename(test)) self.test_kwargs['expected'] = expected self.test_kwargs['oop'] = oop for handler in self.test_handlers: if handler.match(os.path.basename(test)): handler.add_tests_to_suite(mod_name, filepath, suite, testloader, self.marionette, self.testvars, **self.test_kwargs) break if suite.countTestCases(): runner = self.textrunnerclass(verbosity=3, marionette=self.marionette, capabilities=self.capabilities) results = runner.run(suite) self.results.append(results) self.failed += len(results.failures) + len(results.errors) if hasattr(results, 'skipped'): self.todo += len(results.skipped) self.passed += results.passed for failure in results.failures + results.errors: self.failures.append((results.getInfo(failure), failure.output, 'TEST-UNEXPECTED-FAIL')) if hasattr(results, 'unexpectedSuccesses'): self.failed += len(results.unexpectedSuccesses) for failure in results.unexpectedSuccesses: self.failures.append((results.getInfo(failure), 'TEST-UNEXPECTED-PASS')) if hasattr(results, 'expectedFailures'): self.passed += len(results.expectedFailures)
def add_test(self, test, expected='pass', oop=None): filepath = os.path.abspath(test) if os.path.isdir(filepath): for root, dirs, files in os.walk(filepath): for filename in files: if ((filename.startswith('test_') or filename.startswith('browser_')) and (filename.endswith('.py') or filename.endswith('.js'))): filepath = os.path.join(root, filename) self.add_test(filepath) return testargs = {} if self.type is not None: testtypes = self.type.replace('+', ' +').replace('-', ' -').split() for atype in testtypes: if atype.startswith('+'): testargs.update({ atype[1:]: 'true' }) elif atype.startswith('-'): testargs.update({ atype[1:]: 'false' }) else: testargs.update({ atype: 'true' }) # testarg_oop = either None, 'true' or 'false'. testarg_oop = testargs.get('oop') file_ext = os.path.splitext(os.path.split(filepath)[-1])[1] if file_ext == '.ini': manifest = TestManifest() manifest.read(filepath) manifest_tests = manifest.active_tests(exists=False, disabled=True, device=self.device, app=self.appName, **mozinfo.info) unfiltered_tests = [] for test in manifest_tests: if test.get('disabled'): self.manifest_skipped_tests.append(test) else: unfiltered_tests.append(test) # Don't filter tests with "oop" flag because manifest parser can't # handle it well. if testarg_oop is not None: del testargs['oop'] target_tests = manifest.get(tests=unfiltered_tests, **testargs) for test in unfiltered_tests: if test['path'] not in [x['path'] for x in target_tests]: test.setdefault('disabled', 'filtered by type (%s)' % self.type) self.manifest_skipped_tests.append(test) for test in self.manifest_skipped_tests: self.logger.info('TEST-SKIP | %s | %s' % ( os.path.basename(test['path']), test['disabled'])) self.todo += 1 for i in target_tests: if not os.path.exists(i["path"]): raise IOError("test file: %s does not exist" % i["path"]) # manifest_oop is either 'false', 'true' or 'both'. Anything # else implies 'false'. manifest_oop = i.get('oop', 'false') # We only add an oop test when following conditions are met: # 1) It's written by javascript because we have only # MarionetteJSTestCase that supports oop mode. # 2) we're running with "--type=+oop" or no "--type=-oop", which # follows testarg_oop is either None or 'true' and must not # be 'false'. # 3) When no "--type=[+-]oop" is applied, all active tests are # included in target_tests, so we must filter out those # really capable of running in oop mode. Besides, oop tests # must be explicitly specified for backward compatibility. So # test manifest_oop equals to either 'both' or 'true'. file_ext = os.path.splitext(os.path.split(i['path'])[-1])[-1] if (file_ext == '.js' and testarg_oop != 'false' and (manifest_oop == 'both' or manifest_oop == 'true')): self.add_test(i["path"], i["expected"], True) # We only add an in-process test when following conditions are # met: # 1) we're running with "--type=-oop" or no "--type=+oop", which # follows testarg_oop is either None or 'false' and must not # be 'true'. # 2) When no "--type=[+-]oop" is applied, all active tests are # included in target_tests, so we must filter out those # really capable of running in in-process mode. if (testarg_oop != 'true' and (manifest_oop == 'both' or manifest_oop != 'true')): self.add_test(i["path"], i["expected"], False) return if oop is None: # This test is added by directory enumeration or directly specified # in argument list. We have no manifest information here so we just # respect the "--type=[+-]oop" argument here. oop = file_ext == '.js' and testarg_oop == 'true' self.tests.append({'filepath': filepath, 'expected': expected, 'oop': oop})