def run(arguments=sys.argv[1:]):
# parse the command line arguments
(options, command) = parse_args(arguments)
# ensure the binary is given
if not options.binary:
print "Please provide a path to your Firefox binary: -b, --binary"
sys.exit(1)
# Parse the manifest
mp = TestManifest(manifests=(options.manifest,), strict=False)
# run + report
if command == "testpy":
results = test_all_python(mp.get(tests=mp.active_tests(disabled=False), type='python'), options)
if results.failures or results.errors:
sys.exit(report(True, results, None, options))
else:
sys.exit(report(False))
elif command == "testjs":
results = test_all_js(mp.get(tests=mp.active_tests(disabled=False), type='javascript'), options)
if results.failures:
sys.exit(report(True, None, results, options))
else:
sys.exit(report(False))
elif command == "testall":
test_all(mp.active_tests(disabled=False), options)
def run(arguments=sys.argv[1:]):
# parse the command line arguments
parser_kwargs = dict(arguments=arguments)
(options, command) = parse_args(**parser_kwargs)
# Parse the manifest
mp = TestManifest(manifests=(options.manifest,))
# run + report
if command == "testpy":
results = test_all_python(mp.get(tests=mp.active_tests(disabled=False), type='python'), options)
if results.failures or results.errors:
sys.exit(report(True, results, None, options))
else:
sys.exit(report(False))
elif command == "testjs":
results = test_all_js(mp.get(tests=mp.active_tests(disabled=False), type='javascript'), options)
if results.failures:
sys.exit(report(True, None, results, options))
else:
sys.exit(report(False))
elif command == "testall":
test_all(mp.active_tests(disabled=False), options)
def run(arguments=sys.argv[1:]):
# parse the command line arguments
(options, command) = parse_args(arguments)
# ensure the binary is given
if not options.binary:
print "Please provide a path to your Firefox binary: -b, --binary"
sys.exit(1)
# set the BROWSER_PATH environment variable so that
# subshells will be able to invoke mozrunner
os.environ['BROWSER_PATH'] = options.binary
# Parse the manifest
mp = TestManifest(manifests=(options.manifest,), strict=False)
# run + report
if command == "testpy":
tests = mp.active_tests(disabled=False)
results = test_all_python(mp.get(tests=tests, type='python'), options)
if results.failures or results.errors:
sys.exit(report(True, results, None, options))
else:
sys.exit(report(False))
elif command == "testjs":
tests = mp.active_tests(disabled=False)
results = test_all_js(mp.get(tests=tests, type='javascript'), options)
if results.fails:
sys.exit(report(True, None, results, options))
else:
sys.exit(report(False))
elif command == "testall":
test_all(mp.active_tests(disabled=False), options)
def run(arguments=sys.argv[1:]):
# parse the command line arguments
(options, command) = parse_args(arguments)
# ensure the binary is given
if not options.binary:
print "Please provide a path to your Firefox binary: -b, --binary"
sys.exit(1)
# set the BROWSER_PATH environment variable so that
# subshells will be able to invoke mozrunner
os.environ['BROWSER_PATH'] = options.binary
# Parse the manifest
mp = TestManifest(manifests=(options.manifest,), strict=False)
# run + report
if command == "testpy":
tests = mp.active_tests(disabled=False)
results = test_all_python(mp.get(tests=tests, type='python'), options)
if results.failures or results.errors:
sys.exit(report(True, results, None, options))
else:
sys.exit(report(False))
elif command == "testjs":
tests = mp.active_tests(disabled=False)
results = test_all_js(mp.get(tests=tests, type='javascript'), options)
if results.fails:
sys.exit(report(True, None, results, options))
else:
sys.exit(report(False))
elif command == "testall":
test_all(mp.active_tests(disabled=False), options)
def __init__(self, options, **kwargs):
self.options = options
self.server = None
self.logger = mozlog.getLogger('PEP')
# create the profile
self.profile = self.profile_class(
profile=self.options.profilePath,
addons=[os.path.join(here, 'extension')])
# fork a server to serve the test related files
if self.options.serverPath:
self.runServer()
tests = []
# TODO is there a better way of doing this?
if self.options.testPath.endswith('.js'):
# a single test file was passed in
testObj = {}
testObj['path'] = os.path.realpath(self.options.testPath)
testObj['name'] = os.path.basename(self.options.testPath)
testObj['here'] = os.path.dirname(testObj['path'])
tests.append(testObj)
else:
# a test manifest was passed in
# open and convert the manifest to json
manifest = TestManifest()
manifest.read(self.options.testPath)
tests = manifest.get()
# create a manifest object to be read by the JS side
manifestObj = {}
manifestObj['tests'] = tests
# write manifest to a JSON file
jsonManifest = open(os.path.join(here, 'manifest.json'), 'w')
jsonManifest.write(str(manifestObj).replace("'", "\""))
jsonManifest.close()
# setup environment
env = os.environ.copy()
env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1'
env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = str(
options.tracerThreshold)
env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = str(options.tracerInterval)
env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
# construct the browser arguments
cmdargs = []
# TODO Make browserArgs a list
cmdargs.extend(self.options.browserArgs)
cmdargs.extend(['-pep-start', os.path.realpath(jsonManifest.name)])
# run with managed process handler
self.runner = self.runner_class(profile=self.profile,
binary=self.options.binary,
cmdargs=cmdargs,
env=env,
process_class=PepProcess)
def __init__(self, options, **kwargs):
self.options = options
self.server = None
self.logger = mozlog.getLogger('PEP')
# create the profile
self.profile = self.profile_class(profile=self.options.profilePath,
addons=[os.path.join(here, 'extension')])
# fork a server to serve the test related files
if self.options.serverPath:
self.runServer()
tests = []
# TODO is there a better way of doing this?
if self.options.testPath.endswith('.js'):
# a single test file was passed in
testObj = {}
testObj['path'] = os.path.realpath(self.options.testPath)
testObj['name'] = os.path.basename(self.options.testPath)
tests.append(testObj)
else:
# a test manifest was passed in
# open and convert the manifest to json
manifest = TestManifest()
manifest.read(self.options.testPath)
tests = manifest.get()
# create a manifest object to be read by the JS side
manifestObj = {}
manifestObj['tests'] = tests
# write manifest to a JSON file
jsonManifest = open(os.path.join(here, 'manifest.json'), 'w')
jsonManifest.write(str(manifestObj).replace("'", "\""))
jsonManifest.close()
# setup environment
env = os.environ.copy()
env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1'
env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = str(options.tracerThreshold)
env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = str(options.tracerInterval)
env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
# construct the browser arguments
cmdargs = []
# TODO Make browserArgs a list
cmdargs.extend(self.options.browserArgs)
cmdargs.extend(['-pep-start', os.path.realpath(jsonManifest.name)])
# run with managed process handler
self.runner = self.runner_class(profile=self.profile,
binary=self.options.binary,
cmdargs=cmdargs,
env=env,
process_class=PepProcess)
def get_test_list(self, manifest):
self.logger.info("Reading test manifest: %s" % manifest)
mft = TestManifest()
mft.read(manifest)
# In the future if we want to add in more processing to the manifest
# here is where you'd do that. Right now, we just return a list of
# tests
testlist = []
for i in mft.get():
testlist.append(i["path"])
return testlist
def get_test_list(self, manifest):
self.logger.info("Reading test manifest: %s" % manifest)
mft = TestManifest()
mft.read(manifest)
# In the future if we want to add in more processing to the manifest
# here is where you'd do that. Right now, we just return a list of
# tests
testlist = []
for i in mft.get():
testlist.append(i["path"])
return testlist
def read_tests(self):
self._tests = []
manifest = TestManifest()
manifest.read(self._test_path)
tests_info = manifest.get()
for t in tests_info:
if not t['here'] in sys.path:
sys.path.append(t['here'])
if t['name'].endswith('.py'):
t['name'] = t['name'][:-3]
# add all classes in module that are derived from PhoneTest to
# the test list
tests = [(x[1], os.path.normpath(os.path.join(t['here'],
t.get('config', ''))))
for x in inspect.getmembers(__import__(t['name']),
inspect.isclass)
if x[0] != 'PhoneTest' and issubclass(x[1],
phonetest.PhoneTest)]
self._tests.extend(tests)
def print_test_dirs(topsrcdir, manifest_file):
"""
Simple routine which prints the paths of directories specified
in a Marionette manifest, relative to topsrcdir. This does not recurse
into manifests, as we currently have no need for that.
"""
# output the directory of this (parent) manifest
topsrcdir = os.path.abspath(topsrcdir)
scriptdir = os.path.abspath(os.path.dirname(__file__))
print scriptdir[len(topsrcdir) + 1:]
# output the directories of all the other manifests
dirs = set()
manifest = TestManifest()
manifest.read(manifest_file)
for i in manifest.get():
dirs.add(os.path.dirname(i['manifest'])[len(topsrcdir) + 1:])
for path in dirs:
print path
def print_test_dirs(topsrcdir, manifest_file):
"""
Simple routine which prints the paths of directories specified
in a Marionette manifest, relative to topsrcdir. This does not recurse
into manifests, as we currently have no need for that.
"""
dirs = set()
# output the directory of this (parent) manifest
topsrcdir = os.path.abspath(topsrcdir)
scriptdir = os.path.abspath(os.path.dirname(__file__))
dirs.add(scriptdir[len(topsrcdir) + 1:])
# output the directories of all the other manifests
manifest = TestManifest()
manifest.read(manifest_file)
for i in manifest.get():
d = os.path.dirname(i['manifest'])[len(topsrcdir) + 1:]
dirs.add(d)
for path in dirs:
path = path.replace('\\', '/')
print path
def add_test(self, test, expected='pass', test_container=None):
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if (filename.startswith('test_') and
(filename.endswith('.py') or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.add_test(filepath)
return
testargs = {}
if self.type is not None:
testtypes = self.type.replace('+', ' +').replace('-', ' -').split()
for atype in testtypes:
if atype.startswith('+'):
testargs.update({ atype[1:]: 'true' })
elif atype.startswith('-'):
testargs.update({ atype[1:]: 'false' })
else:
testargs.update({ atype: 'true' })
testarg_b2g = bool(testargs.get('b2g'))
file_ext = os.path.splitext(os.path.split(filepath)[-1])[1]
if file_ext == '.ini':
manifest = TestManifest()
manifest.read(filepath)
filters = []
if self.test_tags:
filters.append(tags(self.test_tags))
manifest_tests = manifest.active_tests(exists=False,
disabled=True,
filters=filters,
device=self.device,
app=self.appName,
**mozinfo.info)
if len(manifest_tests) == 0:
self.logger.error("no tests to run using specified "
"combination of filters: {}".format(
manifest.fmt_filters()))
unfiltered_tests = []
for test in manifest_tests:
if test.get('disabled'):
self.manifest_skipped_tests.append(test)
else:
unfiltered_tests.append(test)
target_tests = manifest.get(tests=unfiltered_tests, **testargs)
for test in unfiltered_tests:
if test['path'] not in [x['path'] for x in target_tests]:
test.setdefault('disabled', 'filtered by type (%s)' % self.type)
self.manifest_skipped_tests.append(test)
for i in target_tests:
if not os.path.exists(i["path"]):
raise IOError("test file: %s does not exist" % i["path"])
file_ext = os.path.splitext(os.path.split(i['path'])[-1])[-1]
test_container = None
if i.get('test_container') and testarg_b2g:
if i.get('test_container') == "true":
test_container = True
elif i.get('test_container') == "false":
test_container = False
self.add_test(i["path"], i["expected"], test_container)
return
self.tests.append({'filepath': filepath, 'expected': expected, 'test_container': test_container})
def run_test(self, test, expected='pass'):
if not self.httpd:
print "starting httpd"
self.start_httpd()
if not self.marionette:
self.start_marionette()
if self.emulator:
self.marionette.emulator.wait_for_homescreen(self.marionette)
testargs = {}
if self.type is not None:
testtypes = self.type.replace('+', ' +').replace('-', ' -').split()
for atype in testtypes:
if atype.startswith('+'):
testargs.update({atype[1:]: 'true'})
elif atype.startswith('-'):
testargs.update({atype[1:]: 'false'})
else:
testargs.update({atype: 'true'})
oop = testargs.get('oop', False)
if isinstance(oop, basestring):
oop = False if oop == 'false' else 'true'
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
if self.shuffle:
random.shuffle(files)
for filename in files:
if ((filename.startswith('test_')
or filename.startswith('browser_'))
and (filename.endswith('.py')
or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.run_test(filepath)
if self.marionette.check_for_crash():
return
return
mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1])
testloader = unittest.TestLoader()
suite = unittest.TestSuite()
if file_ext == '.ini':
manifest = TestManifest()
manifest.read(filepath)
all_tests = manifest.active_tests(exists=False, disabled=False)
manifest_tests = manifest.active_tests(exists=False,
disabled=False,
device=self.device,
app=self.appName)
skip_tests = list(
set([x['path'] for x in all_tests]) -
set([x['path'] for x in manifest_tests]))
for skipped in skip_tests:
self.logger.info(
'TEST-SKIP | %s | device=%s, app=%s' %
(os.path.basename(skipped), self.device, self.appName))
self.todo += 1
target_tests = manifest.get(tests=manifest_tests, **testargs)
if self.shuffle:
random.shuffle(target_tests)
for i in target_tests:
self.run_test(i["path"], i["expected"])
if self.marionette.check_for_crash():
return
return
self.logger.info('TEST-START %s' % os.path.basename(test))
self.test_kwargs['expected'] = expected
self.test_kwargs['oop'] = oop
for handler in self.test_handlers:
if handler.match(os.path.basename(test)):
handler.add_tests_to_suite(mod_name, filepath, suite,
testloader, self.marionette,
self.testvars, **self.test_kwargs)
break
if suite.countTestCases():
runner = self.textrunnerclass(verbosity=3,
marionette=self.marionette)
results = runner.run(suite)
self.results.append(results)
self.failed += len(results.failures) + len(results.errors)
if hasattr(results, 'skipped'):
self.todo += len(results.skipped)
self.passed += results.passed
for failure in results.failures + results.errors:
self.failures.append((results.getInfo(failure), failure.output,
'TEST-UNEXPECTED-FAIL'))
if hasattr(results, 'unexpectedSuccesses'):
self.failed += len(results.unexpectedSuccesses)
for failure in results.unexpectedSuccesses:
self.failures.append(
(results.getInfo(failure), 'TEST-UNEXPECTED-PASS'))
if hasattr(results, 'expectedFailures'):
self.passed += len(results.expectedFailures)
def run_test(self, test, testtype):
if not self.httpd:
self.start_httpd()
if not self.marionette:
self.start_marionette()
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if ((filename.startswith('test_') or filename.startswith('browser_')) and
(filename.endswith('.py') or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.run_test(filepath, testtype)
return
mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1])
testloader = unittest.TestLoader()
suite = unittest.TestSuite()
if file_ext == '.ini':
if testtype is not None:
testargs = {}
testtypes = testtype.replace('+', ' +').replace('-', ' -').split()
for atype in testtypes:
if atype.startswith('+'):
testargs.update({ atype[1:]: 'true' })
elif atype.startswith('-'):
testargs.update({ atype[1:]: 'false' })
else:
testargs.update({ atype: 'true' })
manifest = TestManifest()
manifest.read(filepath)
if testtype is None:
manifest_tests = manifest.get()
else:
manifest_tests = manifest.get(**testargs)
for i in manifest_tests:
self.run_test(i["path"], testtype)
return
self.logger.info('TEST-START %s' % os.path.basename(test))
if file_ext == '.py':
test_mod = imp.load_source(mod_name, filepath)
for name in dir(test_mod):
obj = getattr(test_mod, name)
if (isinstance(obj, (type, types.ClassType)) and
issubclass(obj, unittest.TestCase)):
testnames = testloader.getTestCaseNames(obj)
for testname in testnames:
suite.addTest(obj(self.marionette, methodName=testname))
elif file_ext == '.js':
suite.addTest(MarionetteJSTestCase(self.marionette, jsFile=filepath))
if suite.countTestCases():
results = MarionetteTextTestRunner(verbosity=3).run(suite)
self.failed += len(results.failures) + len(results.errors)
self.todo = 0
if hasattr(results, 'skipped'):
self.todo += len(results.skipped) + len(results.expectedFailures)
self.passed += results.passed
for failure in results.failures + results.errors:
self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL'))
if hasattr(results, 'unexpectedSuccess'):
self.failed += len(results.unexpectedSuccesses)
for failure in results.unexpectedSuccesses:
self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-PASS'))
def __init__(self, options, **kwargs):
self.options = options
self.server = None
self.logger = mozlog.getLogger('PEP')
# create the profile
enable_proxy = False
locations = ServerLocations()
if self.options.proxyLocations:
if not self.options.serverPath:
self.logger.warning('Can\'t set up proxy without server path')
else:
enable_proxy = True
for proxyLocation in self.options.proxyLocations:
locations.read(proxyLocation, False)
locations.add_host(host='127.0.0.1',
port=self.options.serverPort,
options='primary,privileged')
self.profile = self.profile_class(
profile=self.options.profilePath,
addons=[os.path.join(here, 'extension')],
locations=locations,
proxy=enable_proxy)
# fork a server to serve the test related files
if self.options.serverPath:
self.runServer()
tests = []
# TODO is there a better way of doing this?
if self.options.testPath.endswith('.js'):
# a single test file was passed in
testObj = {}
testObj['path'] = os.path.realpath(self.options.testPath)
testObj['name'] = os.path.basename(self.options.testPath)
testObj['here'] = os.path.dirname(testObj['path'])
tests.append(testObj)
else:
# a test manifest was passed in
# open and convert the manifest to json
manifest = TestManifest()
manifest.read(self.options.testPath)
tests = manifest.get()
# create a manifest object to be read by the JS side
manifestObj = {}
manifestObj['tests'] = tests
manifestObj['options'] = options.__dict__
# write manifest to a JSON file
jsonManifest = open(os.path.join(here, 'manifest.json'), 'w')
jsonManifest.write(json.dumps(manifestObj))
jsonManifest.close()
# setup environment
env = os.environ.copy()
env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1'
env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = str(
options.tracerThreshold)
env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = str(options.tracerInterval)
env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
# construct the browser arguments
cmdargs = []
# TODO Make browserArgs a list
cmdargs.extend(self.options.browserArgs)
cmdargs.extend(['-pep-start', os.path.realpath(jsonManifest.name)])
# run with managed process handler
self.runner = self.runner_class(profile=self.profile,
binary=self.options.binary,
cmdargs=cmdargs,
env=env,
process_class=PepProcess)
def run_test(self, test, testtype):
if not self.httpd:
print "starting httpd"
self.start_httpd()
if not self.marionette:
self.start_marionette()
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if ((filename.startswith('test_') or filename.startswith('browser_')) and
(filename.endswith('.py') or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.run_test(filepath, testtype)
return
mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1])
testloader = unittest.TestLoader()
suite = unittest.TestSuite()
if file_ext == '.ini':
testargs = { 'skip': 'false' }
if testtype is not None:
testtypes = testtype.replace('+', ' +').replace('-', ' -').split()
for atype in testtypes:
if atype.startswith('+'):
testargs.update({ atype[1:]: 'true' })
elif atype.startswith('-'):
testargs.update({ atype[1:]: 'false' })
else:
testargs.update({ atype: 'true' })
manifest = TestManifest()
manifest.read(filepath)
if self.perf:
if self.perfserv is None:
self.perfserv = manifest.get("perfserv")[0]
machine_name = socket.gethostname()
try:
manifest.has_key("machine_name")
machine_name = manifest.get("machine_name")[0]
except:
self.logger.info("Using machine_name: %s" % machine_name)
os_name = platform.system()
os_version = platform.release()
self.perfrequest = datazilla.DatazillaRequest(
server=self.perfserv,
machine_name=machine_name,
os=os_name,
os_version=os_version,
platform=manifest.get("platform")[0],
build_name=manifest.get("build_name")[0],
version=manifest.get("version")[0],
revision=self.revision,
branch=manifest.get("branch")[0],
id=os.getenv('BUILD_ID'),
test_date=int(time.time()))
manifest_tests = manifest.get(**testargs)
for i in manifest_tests:
self.run_test(i["path"], testtype)
return
self.logger.info('TEST-START %s' % os.path.basename(test))
for handler in self.test_handlers:
if handler.match(os.path.basename(test)):
handler.add_tests_to_suite(mod_name, filepath, suite, testloader, self.marionette)
break
if suite.countTestCases():
results = MarionetteTextTestRunner(verbosity=3, perf=self.perf).run(suite)
self.results.append(results)
self.failed += len(results.failures) + len(results.errors)
if results.perfdata and options.perf:
self.perfrequest.add_datazilla_result(results.perfdata)
if hasattr(results, 'skipped'):
self.todo += len(results.skipped) + len(results.expectedFailures)
self.passed += results.passed
for failure in results.failures + results.errors:
self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL'))
if hasattr(results, 'unexpectedSuccess'):
self.failed += len(results.unexpectedSuccesses)
for failure in results.unexpectedSuccesses:
self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-PASS'))
def run_test(self, test):
if not self.httpd:
print "starting httpd"
self.start_httpd()
if not self.marionette:
self.start_marionette()
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if ((filename.startswith('test_') or filename.startswith('browser_')) and
(filename.endswith('.py') or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.run_test(filepath)
if self.marionette.check_for_crash():
return
return
mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1])
testloader = unittest.TestLoader()
suite = unittest.TestSuite()
if file_ext == '.ini':
testargs = {}
if self.type is not None:
testtypes = self.type.replace('+', ' +').replace('-', ' -').split()
for atype in testtypes:
if atype.startswith('+'):
testargs.update({ atype[1:]: 'true' })
elif atype.startswith('-'):
testargs.update({ atype[1:]: 'false' })
else:
testargs.update({ atype: 'true' })
manifest = TestManifest()
manifest.read(filepath)
all_tests = manifest.active_tests(disabled=False)
manifest_tests = manifest.active_tests(disabled=False,
device=self.device,
app=self.appName)
skip_tests = list(set([x['path'] for x in all_tests]) -
set([x['path'] for x in manifest_tests]))
for skipped in skip_tests:
self.logger.info('TEST-SKIP | %s | device=%s, app=%s' %
(os.path.basename(skipped),
self.device,
self.appName))
self.todo += 1
for i in manifest.get(tests=manifest_tests, **testargs):
self.run_test(i["path"])
if self.marionette.check_for_crash():
return
return
self.logger.info('TEST-START %s' % os.path.basename(test))
for handler in self.test_handlers:
if handler.match(os.path.basename(test)):
handler.add_tests_to_suite(mod_name,
filepath,
suite,
testloader,
self.marionette,
self.testvars,
**self.test_kwargs)
break
if suite.countTestCases():
runner = self.textrunnerclass(verbosity=3,
marionette=self.marionette)
results = runner.run(suite)
self.results.append(results)
self.failed += len(results.failures) + len(results.errors)
if hasattr(results, 'skipped'):
self.todo += len(results.skipped)
self.passed += results.passed
for failure in results.failures + results.errors:
self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL'))
if hasattr(results, 'unexpectedSuccesses'):
self.failed += len(results.unexpectedSuccesses)
for failure in results.unexpectedSuccesses:
self.failures.append((results.getInfo(failure), 'TEST-UNEXPECTED-PASS'))
if hasattr(results, 'expectedFailures'):
self.passed += len(results.expectedFailures)
def run_test(self, test, testtype):
if not self.httpd:
print "starting httpd"
self.start_httpd()
if not self.marionette:
self.start_marionette()
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if ((filename.startswith('test_') or filename.startswith('browser_')) and
(filename.endswith('.py') or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.run_test(filepath, testtype)
if self.marionette.check_for_crash():
return
return
mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1])
testloader = unittest.TestLoader()
suite = unittest.TestSuite()
if file_ext == '.ini':
testargs = {}
if testtype is not None:
testtypes = testtype.replace('+', ' +').replace('-', ' -').split()
for atype in testtypes:
if atype.startswith('+'):
testargs.update({ atype[1:]: 'true' })
elif atype.startswith('-'):
testargs.update({ atype[1:]: 'false' })
else:
testargs.update({ atype: 'true' })
manifest = TestManifest()
manifest.read(filepath)
if self.perf:
if self.perfserv is None:
self.perfserv = manifest.get("perfserv")[0]
machine_name = socket.gethostname()
try:
manifest.has_key("machine_name")
machine_name = manifest.get("machine_name")[0]
except:
self.logger.info("Using machine_name: %s" % machine_name)
os_name = platform.system()
os_version = platform.release()
self.perfrequest = datazilla.DatazillaRequest(
server=self.perfserv,
machine_name=machine_name,
os=os_name,
os_version=os_version,
platform=manifest.get("platform")[0],
build_name=manifest.get("build_name")[0],
version=manifest.get("version")[0],
revision=self.revision,
branch=manifest.get("branch")[0],
id=os.getenv('BUILD_ID'),
test_date=int(time.time()))
manifest_tests = manifest.active_tests(disabled=False)
for i in manifest.get(tests=manifest_tests, **testargs):
self.run_test(i["path"], testtype)
if self.marionette.check_for_crash():
return
return
self.logger.info('TEST-START %s' % os.path.basename(test))
for handler in self.test_handlers:
if handler.match(os.path.basename(test)):
handler.add_tests_to_suite(mod_name, filepath, suite, testloader, self.marionette, self.testvars)
break
if suite.countTestCases():
runner = MarionetteTextTestRunner(verbosity=3,
perf=self.perf,
marionette=self.marionette)
results = runner.run(suite)
self.results.append(results)
self.failed += len(results.failures) + len(results.errors)
if results.perfdata and options.perf:
self.perfrequest.add_datazilla_result(results.perfdata)
if hasattr(results, 'skipped'):
self.todo += len(results.skipped) + len(results.expectedFailures)
self.passed += results.passed
for failure in results.failures + results.errors:
self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL'))
if hasattr(results, 'unexpectedSuccess'):
self.failed += len(results.unexpectedSuccesses)
for failure in results.unexpectedSuccesses:
self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-PASS'))
def __init__(self, options, **kwargs):
self.options = options
self.server = None
self.logger = mozlog.getLogger("PEP")
# create the profile
enable_proxy = False
locations = ServerLocations()
if self.options.proxyLocations:
if not self.options.serverPath:
self.logger.warning("Can't set up proxy without server path")
else:
enable_proxy = True
locations.read(self.options.proxyLocations, False)
locations.add_host(host="127.0.0.1", port=self.options.serverPort, options="primary,privileged")
self.profile = self.profile_class(
profile=self.options.profilePath,
addons=[os.path.join(here, "extension")],
locations=locations,
proxy=enable_proxy,
)
# fork a server to serve the test related files
if self.options.serverPath:
self.runServer()
tests = []
# TODO is there a better way of doing this?
if self.options.testPath.endswith(".js"):
# a single test file was passed in
testObj = {}
testObj["path"] = os.path.realpath(self.options.testPath)
testObj["name"] = os.path.basename(self.options.testPath)
testObj["here"] = os.path.dirname(testObj["path"])
tests.append(testObj)
else:
# a test manifest was passed in
# open and convert the manifest to json
manifest = TestManifest()
manifest.read(self.options.testPath)
tests = manifest.get()
# create a manifest object to be read by the JS side
manifestObj = {}
manifestObj["tests"] = tests
manifestObj["options"] = options.__dict__
# write manifest to a JSON file
jsonManifest = open(os.path.join(here, "manifest.json"), "w")
jsonManifest.write(json.dumps(manifestObj))
jsonManifest.close()
# setup environment
env = os.environ.copy()
env["MOZ_INSTRUMENT_EVENT_LOOP"] = "1"
env["MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD"] = str(options.tracerThreshold)
env["MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL"] = str(options.tracerInterval)
env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
# construct the browser arguments
cmdargs = []
# TODO Make browserArgs a list
cmdargs.extend(self.options.browserArgs)
cmdargs.extend(["-pep-start", os.path.realpath(jsonManifest.name)])
# run with managed process handler
self.runner = self.runner_class(
profile=self.profile, binary=self.options.binary, cmdargs=cmdargs, env=env, process_class=PepProcess
)
def __init__(self, options, **kwargs):
self.options = options
self.server = None
self.logger = mozlog.getLogger('PEP')
# create the profile
enable_proxy = False
locations = ServerLocations()
if self.options.proxyLocations:
if not self.options.serverPath:
self.logger.warning('Can\'t set up proxy without server path')
else:
enable_proxy = True
for proxyLocation in self.options.proxyLocations:
locations.read(proxyLocation, False)
locations.add_host(host='127.0.0.1',
port=self.options.serverPort,
options='primary,privileged')
self.profile = self.profile_class(profile=self.options.profilePath,
addons=[os.path.join(here, 'extension')],
locations=locations,
proxy=enable_proxy)
# fork a server to serve the test related files
if self.options.serverPath:
self.runServer()
tests = []
# TODO is there a better way of doing this?
if self.options.testPath.endswith('.js'):
# a single test file was passed in
testObj = {}
testObj['path'] = os.path.realpath(self.options.testPath)
testObj['name'] = os.path.basename(self.options.testPath)
testObj['here'] = os.path.dirname(testObj['path'])
tests.append(testObj)
else:
# a test manifest was passed in
# open and convert the manifest to json
manifest = TestManifest()
manifest.read(self.options.testPath)
tests = manifest.get()
# create a manifest object to be read by the JS side
manifestObj = {}
manifestObj['tests'] = tests
manifestObj['options'] = options.__dict__
# write manifest to a JSON file
jsonManifest = open(os.path.join(here, 'manifest.json'), 'w')
jsonManifest.write(json.dumps(manifestObj))
jsonManifest.close()
# setup environment
env = os.environ.copy()
env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1'
env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = str(options.tracerThreshold)
env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = str(options.tracerInterval)
env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
# construct the browser arguments
cmdargs = []
# TODO Make browserArgs a list
cmdargs.extend(self.options.browserArgs)
cmdargs.extend(['-pep-start', os.path.realpath(jsonManifest.name)])
# run with managed process handler
self.runner = self.runner_class(profile=self.profile,
binary=self.options.binary,
cmdargs=cmdargs,
env=env,
process_class=PepProcess)
def add_test(self, test, expected='pass', test_container=None):
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if (filename.startswith('test_') and
(filename.endswith('.py') or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.add_test(filepath)
return
testargs = {}
if self.type is not None:
testtypes = self.type.replace('+', ' +').replace('-', ' -').split()
for atype in testtypes:
if atype.startswith('+'):
testargs.update({ atype[1:]: 'true' })
elif atype.startswith('-'):
testargs.update({ atype[1:]: 'false' })
else:
testargs.update({ atype: 'true' })
testarg_b2g = bool(testargs.get('b2g'))
file_ext = os.path.splitext(os.path.split(filepath)[-1])[1]
if file_ext == '.ini':
manifest = TestManifest()
manifest.read(filepath)
filters = []
if self.test_tags:
filters.append(tags(self.test_tags))
json_path = update_mozinfo(filepath)
self.logger.info("mozinfo updated with the following: {}".format(None))
manifest_tests = manifest.active_tests(exists=False,
disabled=True,
filters=filters,
device=self.device,
app=self.appName,
e10s=self.e10s,
**mozinfo.info)
if len(manifest_tests) == 0:
self.logger.error("no tests to run using specified "
"combination of filters: {}".format(
manifest.fmt_filters()))
unfiltered_tests = []
for test in manifest_tests:
if test.get('disabled'):
self.manifest_skipped_tests.append(test)
else:
unfiltered_tests.append(test)
target_tests = manifest.get(tests=unfiltered_tests, **testargs)
for test in unfiltered_tests:
if test['path'] not in [x['path'] for x in target_tests]:
test.setdefault('disabled', 'filtered by type (%s)' % self.type)
self.manifest_skipped_tests.append(test)
for i in target_tests:
if not os.path.exists(i["path"]):
raise IOError("test file: %s does not exist" % i["path"])
file_ext = os.path.splitext(os.path.split(i['path'])[-1])[-1]
test_container = None
if i.get('test_container') and testarg_b2g:
if i.get('test_container') == "true":
test_container = True
elif i.get('test_container') == "false":
test_container = False
self.add_test(i["path"], i["expected"], test_container)
return
self.tests.append({'filepath': filepath, 'expected': expected, 'test_container': test_container})
def run_test(self, test, testtype):
if not self.httpd:
self.start_httpd()
if not self.marionette:
self.start_marionette()
if not os.path.isabs(test):
filepath = os.path.join(os.path.dirname(__file__), test)
else:
filepath = test
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if ((filename.startswith('test_')
or filename.startswith('browser_'))
and (filename.endswith('.py')
or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.run_test(filepath, testtype)
return
mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1])
testloader = unittest.TestLoader()
suite = unittest.TestSuite()
if file_ext == '.ini':
if testtype is not None:
testargs = {}
testtypes = testtype.replace('+', ' +').replace('-',
' -').split()
for atype in testtypes:
if atype.startswith('+'):
testargs.update({atype[1:]: 'true'})
elif atype.startswith('-'):
testargs.update({atype[1:]: 'false'})
else:
testargs.update({atype: 'true'})
manifest = TestManifest()
manifest.read(filepath)
if testtype is None:
manifest_tests = manifest.get()
else:
manifest_tests = manifest.get(**testargs)
for i in manifest_tests:
self.run_test(i["path"], testtype)
return
self.logger.info('TEST-START %s' % os.path.basename(test))
if file_ext == '.py':
test_mod = imp.load_source(mod_name, filepath)
for name in dir(test_mod):
obj = getattr(test_mod, name)
if (isinstance(obj, (type, types.ClassType))
and issubclass(obj, unittest.TestCase)):
testnames = testloader.getTestCaseNames(obj)
for testname in testnames:
suite.addTest(obj(self.marionette,
methodName=testname))
elif file_ext == '.js':
suite.addTest(
MarionetteJSTestCase(self.marionette, jsFile=filepath))
if suite.countTestCases():
results = MarionetteTextTestRunner(verbosity=3).run(suite)
self.failed += len(results.failures) + len(results.errors)
self.todo = 0
if hasattr(results, 'skipped'):
self.todo += len(results.skipped) + len(
results.expectedFailures)
self.passed += results.passed
for failure in results.failures + results.errors:
self.failures.append((results.getInfo(failure[0]), failure[1],
'TEST-UNEXPECTED-FAIL'))
if hasattr(results, 'unexpectedSuccess'):
self.failed += len(results.unexpectedSuccesses)
for failure in results.unexpectedSuccesses:
self.failures.append((results.getInfo(failure[0]),
failure[1], 'TEST-UNEXPECTED-PASS'))
def run_test(self, test):
if not self.httpd:
print "starting httpd"
self.start_httpd()
if not self.marionette:
self.start_marionette()
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if ((filename.startswith('test_')
or filename.startswith('browser_'))
and (filename.endswith('.py')
or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.run_test(filepath)
if self.marionette.check_for_crash():
return
return
mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1])
testloader = unittest.TestLoader()
suite = unittest.TestSuite()
if file_ext == '.ini':
testargs = {}
if self.type is not None:
testtypes = self.type.replace('+', ' +').replace('-',
' -').split()
for atype in testtypes:
if atype.startswith('+'):
testargs.update({atype[1:]: 'true'})
elif atype.startswith('-'):
testargs.update({atype[1:]: 'false'})
else:
testargs.update({atype: 'true'})
manifest = TestManifest()
manifest.read(filepath)
manifest_tests = manifest.active_tests(disabled=False)
for i in manifest.get(tests=manifest_tests, **testargs):
self.run_test(i["path"])
if self.marionette.check_for_crash():
return
return
self.logger.info('TEST-START %s' % os.path.basename(test))
for handler in self.test_handlers:
if handler.match(os.path.basename(test)):
handler.add_tests_to_suite(mod_name, filepath, suite,
testloader, self.marionette,
self.testvars, **self.test_kwargs)
break
if suite.countTestCases():
runner = MarionetteTextTestRunner(verbosity=3,
marionette=self.marionette)
results = runner.run(suite)
self.results.append(results)
self.failed += len(results.failures) + len(results.errors)
if hasattr(results, 'skipped'):
self.todo += len(results.skipped) + len(
results.expectedFailures)
self.passed += results.passed
for failure in results.failures + results.errors:
self.failures.append((results.getInfo(failure[0]), failure[1],
'TEST-UNEXPECTED-FAIL'))
if hasattr(results, 'unexpectedSuccess'):
self.failed += len(results.unexpectedSuccesses)
for failure in results.unexpectedSuccesses:
self.failures.append((results.getInfo(failure[0]),
failure[1], 'TEST-UNEXPECTED-PASS'))
def add_test(self, test, expected="pass", test_container=None):
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if filename.startswith("test_") and (filename.endswith(".py") or filename.endswith(".js")):
filepath = os.path.join(root, filename)
self.add_test(filepath)
return
testargs = {}
if self.type is not None:
testtypes = self.type.replace("+", " +").replace("-", " -").split()
for atype in testtypes:
if atype.startswith("+"):
testargs.update({atype[1:]: "true"})
elif atype.startswith("-"):
testargs.update({atype[1:]: "false"})
else:
testargs.update({atype: "true"})
testarg_b2g = bool(testargs.get("b2g"))
file_ext = os.path.splitext(os.path.split(filepath)[-1])[1]
if file_ext == ".ini":
manifest = TestManifest()
manifest.read(filepath)
filters = []
if self.test_tags:
filters.append(tags(self.test_tags))
manifest_tests = manifest.active_tests(
exists=False, disabled=True, filters=filters, device=self.device, app=self.appName, **mozinfo.info
)
if len(manifest_tests) == 0:
self.logger.error(
"no tests to run using specified " "combination of filters: {}".format(manifest.fmt_filters())
)
unfiltered_tests = []
for test in manifest_tests:
if test.get("disabled"):
self.manifest_skipped_tests.append(test)
else:
unfiltered_tests.append(test)
target_tests = manifest.get(tests=unfiltered_tests, **testargs)
for test in unfiltered_tests:
if test["path"] not in [x["path"] for x in target_tests]:
test.setdefault("disabled", "filtered by type (%s)" % self.type)
self.manifest_skipped_tests.append(test)
for i in target_tests:
if not os.path.exists(i["path"]):
raise IOError("test file: %s does not exist" % i["path"])
file_ext = os.path.splitext(os.path.split(i["path"])[-1])[-1]
test_container = None
if i.get("test_container") and testarg_b2g:
if i.get("test_container") == "true":
test_container = True
elif i.get("test_container") == "false":
test_container = False
self.add_test(i["path"], i["expected"], test_container)
return
self.tests.append({"filepath": filepath, "expected": expected, "test_container": test_container})
def run_test(self, test, testtype):
if not self.httpd:
print "starting httpd"
self.start_httpd()
if not self.marionette:
self.start_marionette()
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if (filename.startswith("test_") or filename.startswith("browser_")) and (
filename.endswith(".py") or filename.endswith(".js")
):
filepath = os.path.join(root, filename)
self.run_test(filepath, testtype)
return
mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1])
testloader = unittest.TestLoader()
suite = unittest.TestSuite()
if file_ext == ".ini":
testargs = {"skip": "false"}
if testtype is not None:
testtypes = testtype.replace("+", " +").replace("-", " -").split()
for atype in testtypes:
if atype.startswith("+"):
testargs.update({atype[1:]: "true"})
elif atype.startswith("-"):
testargs.update({atype[1:]: "false"})
else:
testargs.update({atype: "true"})
manifest = TestManifest()
manifest.read(filepath)
if options.perf:
if options.perfserv is None:
options.perfserv = manifest.get("perfserv")[0]
machine_name = socket.gethostname()
try:
manifest.has_key("machine_name")
machine_name = manifest.get("machine_name")[0]
except:
self.logger.info("Using machine_name: %s" % machine_name)
os_name = platform.system()
os_version = platform.release()
self.perfrequest = datazilla.DatazillaRequest(
server=options.perfserv,
machine_name=machine_name,
os=os_name,
os_version=os_version,
platform=manifest.get("platform")[0],
build_name=manifest.get("build_name")[0],
version=manifest.get("version")[0],
revision=self.revision,
branch=manifest.get("branch")[0],
id=os.getenv("BUILD_ID"),
test_date=int(time.time()),
)
manifest_tests = manifest.get(**testargs)
for i in manifest_tests:
self.run_test(i["path"], testtype)
return
self.logger.info("TEST-START %s" % os.path.basename(test))
if file_ext == ".py":
test_mod = imp.load_source(mod_name, filepath)
for name in dir(test_mod):
obj = getattr(test_mod, name)
if isinstance(obj, (type, types.ClassType)) and issubclass(obj, unittest.TestCase):
testnames = testloader.getTestCaseNames(obj)
for testname in testnames:
suite.addTest(obj(self.marionette, methodName=testname))
elif file_ext == ".js":
suite.addTest(MarionetteJSTestCase(self.marionette, jsFile=filepath))
if suite.countTestCases():
results = MarionetteTextTestRunner(verbosity=3).run(suite)
self.results.append(results)
self.failed += len(results.failures) + len(results.errors)
if results.perfdata and options.perf:
self.perfrequest.add_datazilla_result(results.perfdata)
if hasattr(results, "skipped"):
self.todo += len(results.skipped) + len(results.expectedFailures)
self.passed += results.passed
for failure in results.failures + results.errors:
self.failures.append((results.getInfo(failure[0]), failure[1], "TEST-UNEXPECTED-FAIL"))
if hasattr(results, "unexpectedSuccess"):
self.failed += len(results.unexpectedSuccesses)
for failure in results.unexpectedSuccesses:
self.failures.append((results.getInfo(failure[0]), failure[1], "TEST-UNEXPECTED-PASS"))
def run_test(self, test, expected='pass'):
if not self.httpd:
print "starting httpd"
self.start_httpd()
if not self.marionette:
self.start_marionette()
if self.emulator:
self.marionette.emulator.wait_for_homescreen(self.marionette)
# Retrieve capabilities for later use
if not self._capabilities:
self.capabilities
testargs = {}
if self.type is not None:
testtypes = self.type.replace('+', ' +').replace('-', ' -').split()
for atype in testtypes:
if atype.startswith('+'):
testargs.update({ atype[1:]: 'true' })
elif atype.startswith('-'):
testargs.update({ atype[1:]: 'false' })
else:
testargs.update({ atype: 'true' })
oop = testargs.get('oop', False)
if isinstance(oop, basestring):
oop = False if oop == 'false' else 'true'
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
if self.shuffle:
random.shuffle(files)
for filename in files:
if ((filename.startswith('test_') or filename.startswith('browser_')) and
(filename.endswith('.py') or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.run_test(filepath)
if self.marionette.check_for_crash():
return
return
mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1])
testloader = unittest.TestLoader()
suite = unittest.TestSuite()
if file_ext == '.ini':
manifest = TestManifest()
manifest.read(filepath)
manifest_tests = manifest.active_tests(exists=False,
disabled=True,
device=self.device,
app=self.appName,
**mozinfo.info)
unfiltered_tests = []
for test in manifest_tests:
if test.get('disabled'):
self.manifest_skipped_tests.append(test)
else:
unfiltered_tests.append(test)
target_tests = manifest.get(tests=unfiltered_tests, **testargs)
for test in unfiltered_tests:
if test['path'] not in [x['path'] for x in target_tests]:
test.setdefault('disabled', 'filtered by type (%s)' % self.type)
self.manifest_skipped_tests.append(test)
for test in self.manifest_skipped_tests:
self.logger.info('TEST-SKIP | %s | %s' % (
os.path.basename(test['path']),
test['disabled']))
self.todo += 1
if self.shuffle:
random.shuffle(target_tests)
for i in target_tests:
if not os.path.exists(i["path"]):
raise IOError("test file: %s does not exist" % i["path"])
self.run_test(i["path"], i["expected"])
if self.marionette.check_for_crash():
return
return
self.logger.info('TEST-START %s' % os.path.basename(test))
self.test_kwargs['expected'] = expected
self.test_kwargs['oop'] = oop
for handler in self.test_handlers:
if handler.match(os.path.basename(test)):
handler.add_tests_to_suite(mod_name,
filepath,
suite,
testloader,
self.marionette,
self.testvars,
**self.test_kwargs)
break
if suite.countTestCases():
runner = self.textrunnerclass(verbosity=3,
marionette=self.marionette,
capabilities=self.capabilities)
results = runner.run(suite)
self.results.append(results)
self.failed += len(results.failures) + len(results.errors)
if hasattr(results, 'skipped'):
self.todo += len(results.skipped)
self.passed += results.passed
for failure in results.failures + results.errors:
self.failures.append((results.getInfo(failure), failure.output, 'TEST-UNEXPECTED-FAIL'))
if hasattr(results, 'unexpectedSuccesses'):
self.failed += len(results.unexpectedSuccesses)
for failure in results.unexpectedSuccesses:
self.failures.append((results.getInfo(failure), 'TEST-UNEXPECTED-PASS'))
if hasattr(results, 'expectedFailures'):
self.passed += len(results.expectedFailures)
def add_test(self, test, expected='pass', oop=None):
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if ((filename.startswith('test_') or filename.startswith('browser_')) and
(filename.endswith('.py') or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.add_test(filepath)
return
testargs = {}
if self.type is not None:
testtypes = self.type.replace('+', ' +').replace('-', ' -').split()
for atype in testtypes:
if atype.startswith('+'):
testargs.update({ atype[1:]: 'true' })
elif atype.startswith('-'):
testargs.update({ atype[1:]: 'false' })
else:
testargs.update({ atype: 'true' })
# testarg_oop = either None, 'true' or 'false'.
testarg_oop = testargs.get('oop')
file_ext = os.path.splitext(os.path.split(filepath)[-1])[1]
if file_ext == '.ini':
manifest = TestManifest()
manifest.read(filepath)
manifest_tests = manifest.active_tests(exists=False,
disabled=True,
device=self.device,
app=self.appName,
**mozinfo.info)
unfiltered_tests = []
for test in manifest_tests:
if test.get('disabled'):
self.manifest_skipped_tests.append(test)
else:
unfiltered_tests.append(test)
# Don't filter tests with "oop" flag because manifest parser can't
# handle it well.
if testarg_oop is not None:
del testargs['oop']
target_tests = manifest.get(tests=unfiltered_tests, **testargs)
for test in unfiltered_tests:
if test['path'] not in [x['path'] for x in target_tests]:
test.setdefault('disabled', 'filtered by type (%s)' % self.type)
self.manifest_skipped_tests.append(test)
for test in self.manifest_skipped_tests:
self.logger.info('TEST-SKIP | %s | %s' % (
os.path.basename(test['path']),
test['disabled']))
self.todo += 1
for i in target_tests:
if not os.path.exists(i["path"]):
raise IOError("test file: %s does not exist" % i["path"])
# manifest_oop is either 'false', 'true' or 'both'. Anything
# else implies 'false'.
manifest_oop = i.get('oop', 'false')
# We only add an oop test when following conditions are met:
# 1) It's written by javascript because we have only
# MarionetteJSTestCase that supports oop mode.
# 2) we're running with "--type=+oop" or no "--type=-oop", which
# follows testarg_oop is either None or 'true' and must not
# be 'false'.
# 3) When no "--type=[+-]oop" is applied, all active tests are
# included in target_tests, so we must filter out those
# really capable of running in oop mode. Besides, oop tests
# must be explicitly specified for backward compatibility. So
# test manifest_oop equals to either 'both' or 'true'.
file_ext = os.path.splitext(os.path.split(i['path'])[-1])[-1]
if (file_ext == '.js' and
testarg_oop != 'false' and
(manifest_oop == 'both' or manifest_oop == 'true')):
self.add_test(i["path"], i["expected"], True)
# We only add an in-process test when following conditions are
# met:
# 1) we're running with "--type=-oop" or no "--type=+oop", which
# follows testarg_oop is either None or 'false' and must not
# be 'true'.
# 2) When no "--type=[+-]oop" is applied, all active tests are
# included in target_tests, so we must filter out those
# really capable of running in in-process mode.
if (testarg_oop != 'true' and
(manifest_oop == 'both' or manifest_oop != 'true')):
self.add_test(i["path"], i["expected"], False)
return
if oop is None:
# This test is added by directory enumeration or directly specified
# in argument list. We have no manifest information here so we just
# respect the "--type=[+-]oop" argument here.
oop = file_ext == '.js' and testarg_oop == 'true'
self.tests.append({'filepath': filepath, 'expected': expected, 'oop': oop})
