def test_testmanifest(self):
# Test filtering based on platform:
filter_example = os.path.join(here, "filter-example.ini")
manifest = TestManifest(manifests=(filter_example,), strict=False)
self.assertEqual(
[
i["name"]
for i in manifest.active_tests(os="win", disabled=False, exists=False)
],
["windowstest", "fleem"],
)
self.assertEqual(
[
i["name"]
for i in manifest.active_tests(os="linux", disabled=False, exists=False)
],
["fleem", "linuxtest"],
)
# Look for existing tests. There is only one:
self.assertEqual([i["name"] for i in manifest.active_tests()], ["fleem"])
# You should be able to expect failures:
last = manifest.active_tests(exists=False, toolkit="gtk")[-1]
self.assertEqual(last["name"], "linuxtest")
self.assertEqual(last["expected"], "pass")
last = manifest.active_tests(exists=False, toolkit="cocoa")[-1]
self.assertEqual(last["expected"], "fail")
def run(arguments=sys.argv[1:]):
# parse the command line arguments
parser_kwargs = dict(arguments=arguments)
(options, command) = parse_args(**parser_kwargs)
# Parse the manifest
mp = TestManifest(manifests=(options.manifest,))
# run + report
if command == "testpy":
results = test_all_python(mp.get(tests=mp.active_tests(disabled=False), type='python'), options)
if results.failures or results.errors:
sys.exit(report(True, results, None, options))
else:
sys.exit(report(False))
elif command == "testjs":
results = test_all_js(mp.get(tests=mp.active_tests(disabled=False), type='javascript'), options)
if results.failures:
sys.exit(report(True, None, results, options))
else:
sys.exit(report(False))
elif command == "testall":
test_all(mp.active_tests(disabled=False), options)
def run(arguments=sys.argv[1:]):
# parse the command line arguments
(options, command) = parse_args(arguments)
# ensure the binary is given
if not options.binary:
print "Please provide a path to your Firefox binary: -b, --binary"
sys.exit(1)
# Parse the manifest
mp = TestManifest(manifests=(options.manifest,), strict=False)
# run + report
if command == "testpy":
results = test_all_python(mp.get(tests=mp.active_tests(disabled=False), type='python'), options)
if results.failures or results.errors:
sys.exit(report(True, results, None, options))
else:
sys.exit(report(False))
elif command == "testjs":
results = test_all_js(mp.get(tests=mp.active_tests(disabled=False), type='javascript'), options)
if results.failures:
sys.exit(report(True, None, results, options))
else:
sys.exit(report(False))
elif command == "testall":
test_all(mp.active_tests(disabled=False), options)
def run(arguments=sys.argv[1:]):
# parse the command line arguments
(options, command) = parse_args(arguments)
# ensure the binary is given
if not options.binary:
print "Please provide a path to your Firefox binary: -b, --binary"
sys.exit(1)
# set the BROWSER_PATH environment variable so that
# subshells will be able to invoke mozrunner
os.environ['BROWSER_PATH'] = options.binary
# Parse the manifest
mp = TestManifest(manifests=(options.manifest,), strict=False)
# run + report
if command == "testpy":
tests = mp.active_tests(disabled=False)
results = test_all_python(mp.get(tests=tests, type='python'), options)
if results.failures or results.errors:
sys.exit(report(True, results, None, options))
else:
sys.exit(report(False))
elif command == "testjs":
tests = mp.active_tests(disabled=False)
results = test_all_js(mp.get(tests=tests, type='javascript'), options)
if results.fails:
sys.exit(report(True, None, results, options))
else:
sys.exit(report(False))
elif command == "testall":
test_all(mp.active_tests(disabled=False), options)
def run(arguments=sys.argv[1:]):
# parse the command line arguments
(options, command) = parse_args(arguments)
# ensure the binary is given
if not options.binary:
print "Please provide a path to your Firefox binary: -b, --binary"
sys.exit(1)
# set the BROWSER_PATH environment variable so that
# subshells will be able to invoke mozrunner
os.environ['BROWSER_PATH'] = options.binary
# Parse the manifest
mp = TestManifest(manifests=(options.manifest,), strict=False)
# run + report
if command == "testpy":
tests = mp.active_tests(disabled=False)
results = test_all_python(mp.get(tests=tests, type='python'), options)
if results.failures or results.errors:
sys.exit(report(True, results, None, options))
else:
sys.exit(report(False))
elif command == "testjs":
tests = mp.active_tests(disabled=False)
results = test_all_js(mp.get(tests=tests, type='javascript'), options)
if results.fails:
sys.exit(report(True, None, results, options))
else:
sys.exit(report(False))
elif command == "testall":
test_all(mp.active_tests(disabled=False), options)
def test_manifest_subsuites(self):
"""
test subsuites and conditional subsuites
"""
class AttributeDict(dict):
def __getattr__(self, attr):
return self[attr]
def __setattr__(self, attr, value):
self[attr] = value
relative_path = os.path.join(here, 'subsuite.ini')
manifest = TestManifest(manifests=(relative_path, ))
info = {'foo': 'bar'}
options = {'subsuite': 'bar'}
# 6 tests total
self.assertEquals(len(manifest.active_tests(exists=False, **info)), 6)
# only 3 tests for subsuite bar when foo==bar
self.assertEquals(
len(
manifest.active_tests(exists=False,
options=AttributeDict(options),
**info)), 3)
options = {'subsuite': 'baz'}
other = {'something': 'else'}
# only 1 test for subsuite baz, regardless of conditions
self.assertEquals(
len(
manifest.active_tests(exists=False,
options=AttributeDict(options),
**info)), 1)
self.assertEquals(
len(
manifest.active_tests(exists=False,
options=AttributeDict(options),
**other)), 1)
# 4 tests match when the condition doesn't match (all tests except
# the unconditional subsuite)
info = {'foo': 'blah'}
options = {'subsuite': None}
self.assertEquals(
len(
manifest.active_tests(exists=False,
options=AttributeDict(options),
**info)), 5)
# test for illegal subsuite value
manifest.tests[0][
'subsuite'] = 'subsuite=bar,foo=="bar",type="nothing"'
self.assertRaises(ParseError,
manifest.active_tests,
exists=False,
options=AttributeDict(options),
**info)
def test_unknown_keywords(self):
filter_example = os.path.join(here, 'filter-example.ini')
manifest = TestManifest(manifests=(filter_example,))
with self.assertRaises(ParseError):
# toolkit missing
manifest.active_tests(os='win', disabled=False, exists=False)
with self.assertRaises(ParseError):
# os missing
manifest.active_tests(toolkit='windows', disabled=False, exists=False)
def test_none_and_empty_manifest(self):
"""
Test TestManifest for None and empty manifest, see
https://bugzilla.mozilla.org/show_bug.cgi?id=1087682
"""
none_manifest = TestManifest(manifests=None, strict=False)
self.assertEqual(len(none_manifest.test_paths()), 0)
self.assertEqual(len(none_manifest.active_tests()), 0)
empty_manifest = TestManifest(manifests=[], strict=False)
self.assertEqual(len(empty_manifest.test_paths()), 0)
self.assertEqual(len(empty_manifest.active_tests()), 0)
def test_none_and_empty_manifest(self):
"""
Test TestManifest for None and empty manifest, see
https://bugzilla.mozilla.org/show_bug.cgi?id=1087682
"""
none_manifest = TestManifest(manifests=None, strict=False)
self.assertEqual(len(none_manifest.test_paths()), 0)
self.assertEqual(len(none_manifest.active_tests()), 0)
empty_manifest = TestManifest(manifests=[], strict=False)
self.assertEqual(len(empty_manifest.test_paths()), 0)
self.assertEqual(len(empty_manifest.active_tests()), 0)
def cramtest(command_context,
cram_args=None,
test_paths=None,
test_objects=None):
command_context.activate_virtualenv()
import mozinfo
from manifestparser import TestManifest
if test_objects is None:
from moztest.resolve import TestResolver
resolver = command_context._spawn(TestResolver)
if test_paths:
# If we were given test paths, try to find tests matching them.
test_objects = resolver.resolve_tests(paths=test_paths,
flavor="cram")
else:
# Otherwise just run everything in CRAMTEST_MANIFESTS
test_objects = resolver.resolve_tests(flavor="cram")
if not test_objects:
message = "No tests were collected, check spelling of the test paths."
command_context.log(logging.WARN, "cramtest", {}, message)
return 1
mp = TestManifest()
mp.tests.extend(test_objects)
tests = mp.active_tests(disabled=False, **mozinfo.info)
python = command_context.virtualenv_manager.python_path
cmd = [python, "-m", "cram"] + cram_args + [t["relpath"] for t in tests]
return subprocess.call(cmd, cwd=command_context.topsrcdir)
def cramtest(self, cram_args=None, test_paths=None, test_objects=None):
self._activate_virtualenv()
import mozinfo
from manifestparser import TestManifest
if test_objects is None:
from moztest.resolve import TestResolver
resolver = self._spawn(TestResolver)
if test_paths:
# If we were given test paths, try to find tests matching them.
test_objects = resolver.resolve_tests(paths=test_paths, flavor='cram')
else:
# Otherwise just run everything in CRAMTEST_MANIFESTS
test_objects = resolver.resolve_tests(flavor='cram')
if not test_objects:
message = 'No tests were collected, check spelling of the test paths.'
self.log(logging.WARN, 'cramtest', {}, message)
return 1
mp = TestManifest()
mp.tests.extend(test_objects)
tests = mp.active_tests(disabled=False, **mozinfo.info)
python = self.virtualenv_manager.python_path
cmd = [python, '-m', 'cram'] + cram_args + [t['relpath'] for t in tests]
return subprocess.call(cmd, cwd=self.topsrcdir)
def get_suite_list(self):
'''
Returns a dictionary containing a mapping from suites
to the tests they contain.
:return dict: A dictionary with the following structure: {
"suite_name": [
'testing/raptor/test1',
'testing/raptor/test2'
]
}
'''
if self._suite_list:
return self._suite_list
manifest_path = self.get_manifest_path()
# Get the tests from the manifest
test_manifest = TestManifest([manifest_path], strict=False)
test_list = test_manifest.active_tests(exists=False, disabled=False)
# Parse the tests into the expected dictionary
for test in test_list:
# Get the top-level suite
s = os.path.basename(test["here"])
if s not in self._suite_list:
self._suite_list[s] = []
# Get the individual test
fpath = re.sub(".*testing", "testing", test['manifest'])
if fpath not in self._suite_list[s]:
self._suite_list[s].append(fpath)
return self._suite_list
def buildTestPath(self, options):
""" Build the url path to the specific test harness and test file or directory
Build a manifest of tests to run and write out a json file for the harness to read
"""
if options.manifestFile and os.path.isfile(options.manifestFile):
manifest = TestManifest(strict=False)
manifest.read(options.manifestFile)
# Bug 883858 - return all tests including disabled tests
tests = manifest.active_tests(disabled=False, **mozinfo.info)
paths = []
for test in tests:
tp = test['path'].split(self.getTestRoot(options), 1)[1].strip('/')
# Filter out tests if we are using --test-path
if options.testPath and not tp.startswith(options.testPath):
continue
paths.append({'path': tp})
# Bug 883865 - add this functionality into manifestDestiny
with open('tests.json', 'w') as manifestFile:
manifestFile.write(json.dumps({'tests': paths}))
options.manifestFile = 'tests.json'
testHost = "http://mochi.test:8888"
testURL = ("/").join([testHost, self.TEST_PATH, options.testPath])
if os.path.isfile(os.path.join(self.oldcwd, os.path.dirname(__file__), self.TEST_PATH, options.testPath)) and options.repeat > 0:
testURL = ("/").join([testHost, self.PLAIN_LOOP_PATH])
if options.chrome or options.a11y:
testURL = ("/").join([testHost, self.CHROME_PATH])
elif options.browserChrome:
testURL = "about:blank"
elif options.ipcplugins:
testURL = ("/").join([testHost, self.TEST_PATH, "dom/plugins/test"])
return testURL
def get_test_list(self):
"""
Returns a dictionary containing the tests that are in perftest.ini manifest.
:return dict: A dictionary with the following structure: {
"suite_name": {
'perftest_test1',
'perftest_test2',
},
}
"""
for path in pathlib.Path(self.workspace_dir).rglob("perftest.ini"):
suite_name = re.sub(self.workspace_dir, "", os.path.dirname(path))
# Get the tests from perftest.ini
test_manifest = TestManifest([str(path)], strict=False)
test_list = test_manifest.active_tests(exists=False,
disabled=False)
for test in test_list:
si = ScriptInfo(test["path"])
self.script_infos[si["name"]] = si
self._test_list.setdefault(suite_name,
{}).update({si["name"]: ""})
return self._test_list
def add_test(self, test, expected='pass'):
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if filename.endswith('.ini'):
msg_tmpl = (
"Ignoring manifest '{0}'; running all tests in '{1}'."
" See --help for details.")
relpath = os.path.relpath(os.path.join(root, filename),
filepath)
self.logger.warning(msg_tmpl.format(relpath, filepath))
elif self._is_filename_valid(filename):
test_file = os.path.join(root, filename)
self.add_test(test_file)
return
file_ext = os.path.splitext(os.path.split(filepath)[-1])[1]
if file_ext == '.ini':
manifest = TestManifest()
manifest.read(filepath)
filters = []
if self.test_tags:
filters.append(tags(self.test_tags))
update_mozinfo(filepath)
self.logger.info(
"mozinfo updated with the following: {}".format(None))
self.logger.info("mozinfo is: {}".format(mozinfo.info))
manifest_tests = manifest.active_tests(exists=False,
disabled=True,
filters=filters,
app=self.appName,
e10s=self.e10s,
**mozinfo.info)
if len(manifest_tests) == 0:
self.logger.error("No tests to run using specified "
"combination of filters: {}".format(
manifest.fmt_filters()))
target_tests = []
for test in manifest_tests:
if test.get('disabled'):
self.manifest_skipped_tests.append(test)
else:
target_tests.append(test)
for i in target_tests:
if not os.path.exists(i["path"]):
raise IOError("test file: {} does not exist".format(
i["path"]))
file_ext = os.path.splitext(os.path.split(i['path'])[-1])[-1]
self.add_test(i["path"], i["expected"])
return
self.tests.append({'filepath': filepath, 'expected': expected})
def get_test_list(self):
"""
Returns a dictionary containing the tests that are in perftest.ini manifest.
:return dict: A dictionary with the following structure: {
"suite_name": {
'perftest_test1',
'perftest_test2',
},
}
"""
for path in pathlib.Path(self.workspace_dir).rglob("perftest.ini"):
suite_name = re.sub(self.workspace_dir, "", os.path.dirname(path))
# If the workspace dir doesn't end with a forward-slash,
# the substitution above won't work completely
if suite_name.startswith("/") or suite_name.startswith("\\"):
suite_name = suite_name[1:]
# We have to add new paths to the logger as we search
# because mozperftest tests exist in multiple places in-tree
PerfDocLogger.PATHS.append(suite_name)
# Get the tests from perftest.ini
test_manifest = TestManifest([str(path)], strict=False)
test_list = test_manifest.active_tests(exists=False,
disabled=False)
for test in test_list:
si = ScriptInfo(test["path"])
self.script_infos[si["name"]] = si
self._test_list.setdefault(suite_name,
{}).update({si["name"]: str(path)})
return self._test_list
def getActiveTests(self, manifests, options, testDumpFile=None):
# These prefs will cause reftest.jsm to parse the manifests,
# dump the resulting tests to a file, and exit.
prefs = {
'reftest.manifests': json.dumps(manifests),
'reftest.manifests.dumpTests': testDumpFile or self.testDumpFile,
}
cmdargs = []
self.runApp(options, cmdargs=cmdargs, prefs=prefs)
with open(self.testDumpFile, 'r') as fh:
tests = json.load(fh)
if os.path.isfile(self.testDumpFile):
mozfile.remove(self.testDumpFile)
for test in tests:
# Name and path are expected by manifestparser, but not used in reftest.
test['name'] = test['path'] = test['url1']
mp = TestManifest(strict=False)
mp.tests = tests
filters = []
if options.totalChunks:
filters.append(
mpf.chunk_by_manifest(options.thisChunk, options.totalChunks))
tests = mp.active_tests(exists=False, filters=filters)
return tests
def cramtest(self, cram_args=None, test_paths=None, test_objects=None):
self._activate_virtualenv()
import mozinfo
from manifestparser import TestManifest
if test_objects is None:
from mozbuild.testing import TestResolver
resolver = self._spawn(TestResolver)
if test_paths:
# If we were given test paths, try to find tests matching them.
test_objects = resolver.resolve_tests(paths=test_paths,
flavor='cram')
else:
# Otherwise just run everything in CRAMTEST_MANIFESTS
test_objects = resolver.resolve_tests(flavor='cram')
if not test_objects:
message = 'No tests were collected, check spelling of the test paths.'
self.log(logging.WARN, 'cramtest', {}, message)
return 1
mp = TestManifest()
mp.tests.extend(test_objects)
tests = mp.active_tests(disabled=False, **mozinfo.info)
python = self.virtualenv_manager.python_path
cmd = [python, '-m', 'cram'
] + cram_args + [t['relpath'] for t in tests]
return subprocess.call(cmd, cwd=self.topsrcdir)
def get_browser_test_list(browser_app, run_local):
LOG.info(raptor_ini)
test_manifest = TestManifest([raptor_ini], strict=False)
info = {"app": browser_app, "run_local": run_local}
return test_manifest.active_tests(
exists=False, disabled=False, filters=[filter_app], **info
)
def _get_subtests_from_ini(self, manifest_path, suite_name):
"""
Returns a list of (sub)tests from an ini file containing the test definitions.
:param str manifest_path: path to the ini file
:return list: the list of the tests
"""
desc_exclusion = [
"here", "manifest", "manifest_relpath", "path", "relpath"
]
test_manifest = TestManifest([manifest_path], strict=False)
test_list = test_manifest.active_tests(exists=False, disabled=False)
subtests = {}
for subtest in test_list:
subtests[subtest["name"]] = subtest["manifest"]
description = {}
for key, value in subtest.items():
if key not in desc_exclusion:
description[key] = value
self._descriptions.setdefault(suite_name, []).append(description)
self._descriptions[suite_name].sort(key=lambda item: item["name"])
return subtests
def add_test(self, test, expected='pass', test_container=None):
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if (filename.startswith('test_')
and (filename.endswith('.py')
or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.add_test(filepath)
return
file_ext = os.path.splitext(os.path.split(filepath)[-1])[1]
if file_ext == '.ini':
manifest = TestManifest()
manifest.read(filepath)
filters = []
if self.test_tags:
filters.append(tags(self.test_tags))
json_path = update_mozinfo(filepath)
self.logger.info(
"mozinfo updated with the following: {}".format(None))
manifest_tests = manifest.active_tests(exists=False,
disabled=True,
filters=filters,
device=self.device,
app=self.appName,
e10s=self.e10s,
**mozinfo.info)
if len(manifest_tests) == 0:
self.logger.error("no tests to run using specified "
"combination of filters: {}".format(
manifest.fmt_filters()))
target_tests = []
for test in manifest_tests:
if test.get('disabled'):
self.manifest_skipped_tests.append(test)
else:
target_tests.append(test)
for i in target_tests:
if not os.path.exists(i["path"]):
raise IOError("test file: %s does not exist" % i["path"])
file_ext = os.path.splitext(os.path.split(i['path'])[-1])[-1]
test_container = None
self.add_test(i["path"], i["expected"], test_container)
return
self.tests.append({
'filepath': filepath,
'expected': expected,
'test_container': test_container
})
def __call__(self, parser, namespace, values, option_string=None):
from manifestparser import TestManifest
here = os.path.abspath(os.path.dirname(__file__))
raptor_ini = os.path.join(here, 'raptor.ini')
for _app in ["firefox", "chrome", "geckoview", "chrome-android"]:
test_manifest = TestManifest([raptor_ini], strict=False)
info = {"app": _app}
available_tests = test_manifest.active_tests(exists=False,
disabled=False,
filters=[self.filter_app],
**info)
if len(available_tests) == 0:
# none for that app, skip to next
continue
# print in readable format
if _app == "firefox":
title = "\nRaptor Tests Available for %s" % self.get_long_name(_app)
else:
title = "\nRaptor Tests Available for %s (--app=%s)" \
% (self.get_long_name(_app), _app)
print(title)
print("=" * (len(title) - 1))
# build the list of tests for this app
test_list = {}
for next_test in available_tests:
if next_test.get("name", None) is None:
# no test name, skip it
continue
suite = os.path.basename(next_test['manifest'])[:-4]
if suite not in test_list:
test_list[suite] = {'type': None, 'subtests': []}
# for page-load tests we want to list every subtest, so we
# can see which pages are available in which tp6-* sets
if next_test.get("type", None) is not None:
test_list[suite]['type'] = next_test['type']
if next_test['type'] == "pageload":
test_list[suite]['subtests'].append(next_test['name'])
# print the list in a nice readable format
for key in sorted(test_list.iterkeys()):
print("\n%s" % key)
print(" type: %s" % test_list[key]['type'])
if len(test_list[key]['subtests']) != 0:
print(" subtests:")
for _sub in sorted(test_list[key]['subtests']):
print(" %s" % _sub)
print("\nDone.")
# exit Raptor
parser.exit()
def add_test(self, test, expected='pass', test_container=None):
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if (filename.endswith('.ini')):
msg_tmpl = ("Ignoring manifest '{0}'; running all tests in '{1}'."
" See --help for details.")
relpath = os.path.relpath(os.path.join(root, filename), filepath)
self.logger.warning(msg_tmpl.format(relpath, filepath))
elif (filename.startswith('test_') and
(filename.endswith('.py') or filename.endswith('.js'))):
test_file = os.path.join(root, filename)
self.add_test(test_file)
return
file_ext = os.path.splitext(os.path.split(filepath)[-1])[1]
if file_ext == '.ini':
manifest = TestManifest()
manifest.read(filepath)
filters = []
if self.test_tags:
filters.append(tags(self.test_tags))
json_path = update_mozinfo(filepath)
self.logger.info("mozinfo updated with the following: {}".format(None))
manifest_tests = manifest.active_tests(exists=False,
disabled=True,
filters=filters,
app=self.appName,
e10s=self.e10s,
**mozinfo.info)
if len(manifest_tests) == 0:
self.logger.error("no tests to run using specified "
"combination of filters: {}".format(
manifest.fmt_filters()))
target_tests = []
for test in manifest_tests:
if test.get('disabled'):
self.manifest_skipped_tests.append(test)
else:
target_tests.append(test)
for i in target_tests:
if not os.path.exists(i["path"]):
raise IOError("test file: %s does not exist" % i["path"])
file_ext = os.path.splitext(os.path.split(i['path'])[-1])[-1]
test_container = None
self.add_test(i["path"], i["expected"], test_container)
return
self.tests.append({'filepath': filepath, 'expected': expected, 'test_container': test_container})
def test_manifest_subsuites(self):
"""
test subsuites and conditional subsuites
"""
class AttributeDict(dict):
def __getattr__(self, attr):
return self[attr]
def __setattr__(self, attr, value):
self[attr] = value
relative_path = os.path.join(here, 'subsuite.ini')
manifest = TestManifest(manifests=(relative_path,))
info = {'foo': 'bar'}
options = {'subsuite': 'bar'}
# 6 tests total
self.assertEquals(len(manifest.active_tests(exists=False, **info)), 6)
# only 3 tests for subsuite bar when foo==bar
self.assertEquals(len(manifest.active_tests(exists=False,
options=AttributeDict(options),
**info)), 3)
options = {'subsuite': 'baz'}
other = {'something': 'else'}
# only 1 test for subsuite baz, regardless of conditions
self.assertEquals(len(manifest.active_tests(exists=False,
options=AttributeDict(options),
**info)), 1)
self.assertEquals(len(manifest.active_tests(exists=False,
options=AttributeDict(options),
**other)), 1)
# 4 tests match when the condition doesn't match (all tests except
# the unconditional subsuite)
info = {'foo': 'blah'}
options = {'subsuite': None}
self.assertEquals(len(manifest.active_tests(exists=False,
options=AttributeDict(options),
**info)), 5)
# test for illegal subsuite value
manifest.tests[0]['subsuite'] = 'subsuite=bar,foo=="bar",type="nothing"'
self.assertRaises(ParseError, manifest.active_tests, exists=False,
options=AttributeDict(options), **info)
def test_manifest_subsuites(self):
"""
test subsuites and conditional subsuites
"""
relative_path = os.path.join(here, "subsuite.ini")
manifest = TestManifest(manifests=(relative_path,))
info = {"foo": "bar"}
# 6 tests total
tests = manifest.active_tests(exists=False, **info)
self.assertEqual(len(tests), 6)
# only 3 tests for subsuite bar when foo==bar
tests = manifest.active_tests(exists=False, filters=[subsuite("bar")], **info)
self.assertEqual(len(tests), 3)
# only 1 test for subsuite baz, regardless of conditions
other = {"something": "else"}
tests = manifest.active_tests(exists=False, filters=[subsuite("baz")], **info)
self.assertEqual(len(tests), 1)
tests = manifest.active_tests(exists=False, filters=[subsuite("baz")], **other)
self.assertEqual(len(tests), 1)
# 4 tests match when the condition doesn't match (all tests except
# the unconditional subsuite)
info = {"foo": "blah"}
tests = manifest.active_tests(exists=False, filters=[subsuite()], **info)
self.assertEqual(len(tests), 5)
# test for illegal subsuite value
manifest.tests[0]["subsuite"] = 'subsuite=bar,foo=="bar",type="nothing"'
with self.assertRaises(ParseError):
manifest.active_tests(exists=False, filters=[subsuite("foo")], **info)
def test_testmanifest(self):
# Test filtering based on platform:
filter_example = os.path.join(here, 'filter-example.ini')
manifest = TestManifest(manifests=(filter_example,), strict=False)
self.assertEqual([i['name'] for i in manifest.active_tests(os='win', disabled=False, exists=False)],
['windowstest', 'fleem'])
self.assertEqual([i['name'] for i in manifest.active_tests(os='linux', disabled=False, exists=False)],
['fleem', 'linuxtest'])
# Look for existing tests. There is only one:
self.assertEqual([i['name'] for i in manifest.active_tests()],
['fleem'])
# You should be able to expect failures:
last = manifest.active_tests(exists=False, toolkit='gtk2')[-1]
self.assertEqual(last['name'], 'linuxtest')
self.assertEqual(last['expected'], 'pass')
last = manifest.active_tests(exists=False, toolkit='cocoa')[-1]
self.assertEqual(last['expected'], 'fail')
def test_testmanifest(self):
# Test filtering based on platform:
filter_example = os.path.join(here, 'filter-example.ini')
manifest = TestManifest(manifests=(filter_example,))
self.assertEqual([i['name'] for i in manifest.active_tests(os='win', disabled=False, exists=False)],
['windowstest', 'fleem'])
self.assertEqual([i['name'] for i in manifest.active_tests(os='linux', disabled=False, exists=False)],
['fleem', 'linuxtest'])
# Look for existing tests. There is only one:
self.assertEqual([i['name'] for i in manifest.active_tests()],
['fleem'])
# You should be able to expect failures:
last_test = manifest.active_tests(exists=False, toolkit='gtk2')[-1]
self.assertEqual(last_test['name'], 'linuxtest')
self.assertEqual(last_test['expected'], 'pass')
last_test = manifest.active_tests(exists=False, toolkit='cocoa')[-1]
self.assertEqual(last_test['expected'], 'fail')
def add_test(self, test, expected="pass", test_container=None):
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if filename.startswith("test_") and (filename.endswith(".py") or filename.endswith(".js")):
filepath = os.path.join(root, filename)
self.add_test(filepath)
return
file_ext = os.path.splitext(os.path.split(filepath)[-1])[1]
if file_ext == ".ini":
manifest = TestManifest()
manifest.read(filepath)
filters = []
if self.test_tags:
filters.append(tags(self.test_tags))
json_path = update_mozinfo(filepath)
self.logger.info("mozinfo updated with the following: {}".format(None))
manifest_tests = manifest.active_tests(
exists=False,
disabled=True,
filters=filters,
device=self.device,
app=self.appName,
e10s=self.e10s,
**mozinfo.info
)
if len(manifest_tests) == 0:
self.logger.error(
"no tests to run using specified " "combination of filters: {}".format(manifest.fmt_filters())
)
target_tests = []
for test in manifest_tests:
if test.get("disabled"):
self.manifest_skipped_tests.append(test)
else:
target_tests.append(test)
for i in target_tests:
if not os.path.exists(i["path"]):
raise IOError("test file: %s does not exist" % i["path"])
file_ext = os.path.splitext(os.path.split(i["path"])[-1])[-1]
test_container = None
self.add_test(i["path"], i["expected"], test_container)
return
self.tests.append({"filepath": filepath, "expected": expected, "test_container": test_container})
def _get_subtests_from_ini(self, manifest_path):
'''
Returns a list of (sub)tests from an ini file containing the test definitions.
:param str manifest_path: path to the ini file
:return list: the list of the tests
'''
test_manifest = TestManifest([manifest_path], strict=False)
test_list = test_manifest.active_tests(exists=False, disabled=False)
subtest_list = [subtest["name"] for subtest in test_list]
return subtest_list
def get_test_list(self, manifest):
self.logger.info("Reading test manifest: %s" % manifest)
mft = TestManifest()
mft.read(manifest)
# In the future if we want to add in more processing to the manifest
# here is where you'd do that. Right now, we just return a list of
# tests
testlist = []
for i in mft.active_tests(exists=False, disabled=False):
testlist.append(i["path"])
return testlist
def get_test_list(self, manifest):
self.logger.info("Reading test manifest: %s" % manifest)
mft = TestManifest()
mft.read(manifest)
# In the future if we want to add in more processing to the manifest
# here is where you'd do that. Right now, we just return a list of
# tests
testlist = []
for i in mft.active_tests(exists=False, disabled=False):
testlist.append(i["path"])
return testlist
def __call__(self, manifests, buildconfig):
global logger
logger = logger or structuredlog.get_default_logger()
if not isinstance(manifests, Iterable):
manifests = [manifests]
m = TestManifest(manifests)
active = [t['path'] for t in m.active_tests(exists=False, disabled=False, **buildconfig)]
skipped = [t['path'] for t in m.tests if t['path'] not in active]
return active, skipped
def runTests(self):
self.startup()
if isinstance(self.options.manifestFile, TestManifest):
mp = self.options.manifestFile
else:
mp = TestManifest(strict=False)
mp.read(self.options.robocopIni)
filters = []
if self.options.totalChunks:
filters.append(
chunk_by_slice(self.options.thisChunk,
self.options.totalChunks))
robocop_tests = mp.active_tests(exists=False,
filters=filters,
**mozinfo.info)
if not self.options.autorun:
# Force a single loop iteration. The iteration will start Fennec and
# the httpd server, but not actually run a test.
self.options.test_paths = [robocop_tests[0]['name']]
active_tests = []
for test in robocop_tests:
if self.options.test_paths and test[
'name'] not in self.options.test_paths:
continue
if 'disabled' in test:
self.log.info('TEST-INFO | skipping %s | %s' %
(test['name'], test['disabled']))
continue
active_tests.append(test)
tests_by_manifest = defaultdict(list)
for test in active_tests:
tests_by_manifest[test['manifest']].append(test['name'])
self.log.suite_start(tests_by_manifest)
worstTestResult = None
for test in active_tests:
result = self.runSingleTest(test)
if worstTestResult is None or worstTestResult == 0:
worstTestResult = result
if worstTestResult is None:
self.log.warning(
"No tests run. Did you pass an invalid TEST_PATH?")
worstTestResult = 1
else:
print "INFO | runtests.py | Test summary: start."
logResult = self.logTestSummary()
print "INFO | runtests.py | Test summary: end."
if worstTestResult == 0:
worstTestResult = logResult
return worstTestResult
def add_test(self, test, expected='pass', test_container=None):
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if (filename.startswith('test_') and
(filename.endswith('.py') or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.add_test(filepath)
return
file_ext = os.path.splitext(os.path.split(filepath)[-1])[1]
if file_ext == '.ini':
manifest = TestManifest()
manifest.read(filepath)
filters = []
if self.test_tags:
filters.append(tags(self.test_tags))
manifest_tests = manifest.active_tests(exists=False,
disabled=True,
filters=filters,
e10s=self.e10s,
**mozinfo.info)
if len(manifest_tests) == 0:
self.logger.error("no tests to run using specified "
"combination of filters: {}".format(
manifest.fmt_filters()))
target_tests = []
for test in manifest_tests:
if test.get('disabled'):
self.manifest_skipped_tests.append(test)
else:
target_tests.append(test)
for i in target_tests:
if not os.path.exists(i["path"]):
raise IOError("test file: %s does not exist" % i["path"])
file_ext = os.path.splitext(os.path.split(i['path'])[-1])[-1]
test_container = None
self.add_test(i["path"], i["expected"], test_container)
return
self.tests.append({'filepath': filepath, 'expected': expected, 'test_container': test_container})
def convert_ini_manifest_to_json(cls, manifest_path):
manifest = TestManifest([manifest_path])
whitelist = [t['path'] for t in manifest.active_tests(disabled=False)]
blacklist = [t for t in manifest.paths() if t not in whitelist]
whitelist.insert(0, os.path.join(gaia_dir, 'shared', 'test', 'integration', 'setup.js'))
map(lambda l: [os.path.relpath(p, gaia_dir) for p in l] , (whitelist, blacklist))
contents = { 'whitelist': whitelist }
manifest_path = tempfile.NamedTemporaryFile(suffix='.json').name
with open(manifest_path, 'w') as f:
f.writelines(json.dumps(contents, indent=2))
return manifest_path
def runTests(self):
self.startup()
if isinstance(self.options.manifestFile, TestManifest):
mp = self.options.manifestFile
else:
mp = TestManifest(strict=False)
mp.read(self.options.robocopIni)
filters = []
if self.options.totalChunks:
filters.append(
chunk_by_slice(self.options.thisChunk, self.options.totalChunks))
robocop_tests = mp.active_tests(
exists=False, filters=filters, **mozinfo.info)
if not self.options.autorun:
# Force a single loop iteration. The iteration will start Fennec and
# the httpd server, but not actually run a test.
self.options.test_paths = [robocop_tests[0]['name']]
active_tests = []
for test in robocop_tests:
if self.options.test_paths and test['name'] not in self.options.test_paths:
continue
if 'disabled' in test:
self.log.info('TEST-INFO | skipping %s | %s' %
(test['name'], test['disabled']))
continue
active_tests.append(test)
tests_by_manifest = defaultdict(list)
for test in active_tests:
tests_by_manifest[test['manifest']].append(test['name'])
self.log.suite_start(tests_by_manifest)
worstTestResult = None
for test in active_tests:
result = self.runSingleTest(test)
if worstTestResult is None or worstTestResult == 0:
worstTestResult = result
if worstTestResult is None:
self.log.warning(
"No tests run. Did you pass an invalid TEST_PATH?")
worstTestResult = 1
else:
print "INFO | runtests.py | Test summary: start."
logResult = self.logTestSummary()
print "INFO | runtests.py | Test summary: end."
if worstTestResult == 0:
worstTestResult = logResult
return worstTestResult
def convert_ini_manifest_to_json(cls, manifest_path):
manifest = TestManifest([manifest_path])
whitelist = [t['path'] for t in manifest.active_tests(disabled=False)]
blacklist = [t for t in manifest.paths() if t not in whitelist]
whitelist.insert(
0,
os.path.join(gaia_dir, 'shared', 'test', 'integration',
'setup.js'))
map(lambda l: [os.path.relpath(p, gaia_dir) for p in l],
(whitelist, blacklist))
contents = {'whitelist': whitelist}
manifest_path = tempfile.NamedTemporaryFile(suffix='.json').name
with open(manifest_path, 'w') as f:
f.writelines(json.dumps(contents, indent=2))
return manifest_path
def get_manifests(self, suite, mozinfo):
mozinfo = dict(mozinfo)
# Compute all tests for the given suite/subsuite.
tests = self.get_tests(suite)
if "web-platform-tests" in suite:
manifests = set()
for t in tests:
manifests.add(t["manifest"])
return {"active": list(manifests), "skipped": []}
manifests = set(chunk_by_runtime.get_manifest(t) for t in tests)
# Compute the active tests.
m = TestManifest()
m.tests = tests
tests = m.active_tests(disabled=False, exists=False, **mozinfo)
active = set(chunk_by_runtime.get_manifest(t) for t in tests)
skipped = manifests - active
return {"active": list(active), "skipped": list(skipped)}
def get_chunked_manifests(flavor, subsuite, chunks, mozinfo):
"""Compute which manifests should run in which chunks with the given category
of tests.
Args:
flavor (str): The suite to run. Values are defined by the 'build_flavor' key
in `moztest.resolve.TEST_SUITES`.
subsuite (str): The subsuite to run or 'undefined' to denote no subsuite.
chunks (int): Number of chunks to split manifests across.
mozinfo (frozenset): Set of data in the form of (<key>, <value>) used
for filtering.
Returns:
A list of manifests where each item contains the manifest that should
run in the corresponding chunk.
"""
mozinfo = dict(mozinfo)
# Compute all tests for the given suite/subsuite.
tests = get_tests(flavor, subsuite)
all_manifests = set(t['manifest_relpath'] for t in tests)
# Compute only the active tests.
m = TestManifest()
m.tests = tests
tests = m.active_tests(disabled=False, exists=False, **mozinfo)
active_manifests = set(t['manifest_relpath'] for t in tests)
# Run the chunking algorithm.
chunked_manifests = [
c[1] for c in chunk_by_runtime(
None,
chunks,
get_runtimes(mozinfo['os'])
).get_chunked_manifests(tests)
]
# Add all skipped manifests to the first chunk so they still show up in the
# logs. They won't impact runtime much.
skipped_manifests = all_manifests - active_manifests
chunked_manifests[0].extend(skipped_manifests)
return chunked_manifests
def _get_subtests_from_ini(self, manifest_path, suite_name):
"""
Returns a list of (sub)tests from an ini file containing the test definitions.
:param str manifest_path: path to the ini file
:return list: the list of the tests
"""
test_manifest = TestManifest([manifest_path], strict=False)
test_list = test_manifest.active_tests(exists=False, disabled=False)
subtest_list = {}
for subtest in test_list:
subtest_list[subtest["name"]] = subtest["manifest"]
self._urls[subtest["name"]] = {
"type": suite_name,
"url": subtest["test_url"],
}
self._urls = collections.OrderedDict(
sorted(self._urls.items(), key=lambda t: len(t[0])))
return subtest_list
def test_manifest_subsuites(self):
"""
test subsuites and conditional subsuites
"""
relative_path = os.path.join(here, 'subsuite.ini')
manifest = TestManifest(manifests=(relative_path,))
info = {'foo': 'bar'}
# 6 tests total
tests = manifest.active_tests(exists=False, **info)
self.assertEquals(len(tests), 6)
# only 3 tests for subsuite bar when foo==bar
tests = manifest.active_tests(exists=False,
filters=[subsuite('bar')],
**info)
self.assertEquals(len(tests), 3)
# only 1 test for subsuite baz, regardless of conditions
other = {'something': 'else'}
tests = manifest.active_tests(exists=False,
filters=[subsuite('baz')],
**info)
self.assertEquals(len(tests), 1)
tests = manifest.active_tests(exists=False,
filters=[subsuite('baz')],
**other)
self.assertEquals(len(tests), 1)
# 4 tests match when the condition doesn't match (all tests except
# the unconditional subsuite)
info = {'foo': 'blah'}
tests = manifest.active_tests(exists=False,
filters=[subsuite()],
**info)
self.assertEquals(len(tests), 5)
# test for illegal subsuite value
manifest.tests[0]['subsuite'] = 'subsuite=bar,foo=="bar",type="nothing"'
with self.assertRaises(ParseError):
manifest.active_tests(exists=False,
filters=[subsuite('foo')],
**info)
def buildTestPath(self, options):
""" Build the url path to the specific test harness and test file or directory
Build a manifest of tests to run and write out a json file for the harness to read
"""
if options.manifestFile and os.path.isfile(options.manifestFile):
manifest = TestManifest(strict=False)
manifest.read(options.manifestFile)
# Bug 883858 - return all tests including disabled tests
tests = manifest.active_tests(disabled=False, **mozinfo.info)
paths = []
for test in tests:
tp = test['path'].split(self.getTestRoot(options),
1)[1].strip('/')
# Filter out tests if we are using --test-path
if options.testPath and not tp.startswith(options.testPath):
continue
paths.append({'path': tp})
# Bug 883865 - add this functionality into manifestDestiny
with open('tests.json', 'w') as manifestFile:
manifestFile.write(json.dumps({'tests': paths}))
options.manifestFile = 'tests.json'
testHost = "http://mochi.test:8888"
testURL = ("/").join([testHost, self.TEST_PATH, options.testPath])
if os.path.isfile(
os.path.join(self.oldcwd, os.path.dirname(__file__),
self.TEST_PATH,
options.testPath)) and options.repeat > 0:
testURL = ("/").join([testHost, self.PLAIN_LOOP_PATH])
if options.chrome or options.a11y:
testURL = ("/").join([testHost, self.CHROME_PATH])
elif options.browserChrome:
testURL = "about:blank"
elif options.ipcplugins:
testURL = ("/").join(
[testHost, self.TEST_PATH, "dom/plugins/test"])
return testURL
def run_mozmill(runner, args):
tests = []
for test in args.mozmill:
testpath = os.path.expanduser(test)
realpath = os.path.realpath(testpath)
if not os.path.exists(testpath):
raise Exception("Not a valid test file/directory: %s" % test)
root,ext = os.path.splitext(testpath)
if ext == ".ini":
# This is a test manifest, use the parser instead
manifest = TestManifest(manifests=[testpath], strict=False)
print mozinfo.info
tests.extend(manifest.active_tests(**mozinfo.info))
else:
def testname(t):
if os.path.isdir(realpath):
return os.path.join(test, os.path.relpath(t, testpath))
return test
tests.extend([{'name': testname(t), 'path': t }
for t in mozmill.collect_tests(testpath)])
if args.verbose and len(tests):
print "Running these tests:"
print "\t" + "\n\t".join(map(lambda x: x['path'], tests))
exception = None
try:
runner.run(tests, True)
except:
exception_type, exception, tb = sys.exc_info()
results = runner.finish(fatal=exception is not None)
if exception:
traceback.print_exception(exception_type, exception, tb)
if exception or results.fails:
sys.exit(1)
def _get_subtests_from_ini(self, manifest_path, suite_name):
"""
Returns a list of (sub)tests from an ini file containing the test definitions.
:param str manifest_path: path to the ini file
:return list: the list of the tests
"""
test_manifest = TestManifest([manifest_path], strict=False)
test_list = test_manifest.active_tests(exists=False, disabled=False)
subtests = {}
for subtest in test_list:
subtests[subtest["name"]] = subtest["manifest"]
self._urls.setdefault(suite_name, []).append(
{
"test_name": subtest["name"],
"url": subtest["test_url"],
}
)
self._urls[suite_name].sort(key=lambda item: item["test_name"])
return subtests
def parse_manifests(self, tests_path, options=None):
"""Parses a list of given files as manifests, skipping those that are unparsable.
Outputs a summary that gives information about the tests activated/skipped."""
self.total_tests_count = 0
self.skipped_tests_count = 0
options = options or dict()
for rel_path in CodeRevision.MANIFESTS_REL_PATHS:
manifest_path = os.path.join(tests_path, rel_path)
test_manifest = TestManifest([manifest_path])
active_tests = [t['path'] for t in test_manifest.active_tests(exists=False, disabled=False, **options)]
skipped_tests = [t['path'] for t in test_manifest.tests if t['path'] not in active_tests]
self.total_tests_count += len(test_manifest.tests)
self.skipped_tests_count += len(skipped_tests)
test_suite_state = TestSuiteState(revision=self, test_suite=rel_path, options=options,
active_tests=active_tests, skipped_tests=skipped_tests)
self.manifest_states.append(test_suite_state)
test_suite_state.save()
self.processed = True
self.save()
def get_manifests(self, flavor, subsuite, mozinfo):
mozinfo = dict(mozinfo)
# Compute all tests for the given suite/subsuite.
tests = self.get_tests(flavor, subsuite)
if flavor == "web-platform-tests":
manifests = set()
for t in tests:
group = self.get_wpt_group(t)
wpt_group_translation[t['manifest']].add(group)
manifests.add(t['manifest'])
return {"active": list(manifests), "skipped": []}
manifests = set(chunk_by_runtime.get_manifest(t) for t in tests)
# Compute the active tests.
m = TestManifest()
m.tests = tests
tests = m.active_tests(disabled=False, exists=False, **mozinfo)
active = set(chunk_by_runtime.get_manifest(t) for t in tests)
skipped = manifests - active
return {"active": list(active), "skipped": list(skipped)}
def add_test(self, test, expected='pass'):
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if filename.endswith('.ini'):
msg_tmpl = ("Ignoring manifest '{0}'; running all tests in '{1}'."
" See --help for details.")
relpath = os.path.relpath(os.path.join(root, filename), filepath)
self.logger.warning(msg_tmpl.format(relpath, filepath))
elif self._is_filename_valid(filename):
test_file = os.path.join(root, filename)
self.add_test(test_file)
return
file_ext = os.path.splitext(os.path.split(filepath)[-1])[1]
if file_ext == '.ini':
manifest = TestManifest()
manifest.read(filepath)
json_path = update_mozinfo(filepath)
self.logger.info("mozinfo updated from: {}".format(json_path))
self.logger.info("mozinfo is: {}".format(mozinfo.info))
filters = []
if self.test_tags:
filters.append(tags(self.test_tags))
values = {
"appname": self.appName,
"e10s": self.e10s,
"manage_instance": self.marionette.instance is not None,
}
values.update(mozinfo.info)
manifest_tests = manifest.active_tests(exists=False,
disabled=True,
filters=filters,
**values)
if len(manifest_tests) == 0:
self.logger.error("No tests to run using specified "
"combination of filters: {}".format(
manifest.fmt_filters()))
target_tests = []
for test in manifest_tests:
if test.get('disabled'):
self.manifest_skipped_tests.append(test)
else:
target_tests.append(test)
for i in target_tests:
if not os.path.exists(i["path"]):
raise IOError("test file: {} does not exist".format(i["path"]))
file_ext = os.path.splitext(os.path.split(i['path'])[-1])[-1]
self.add_test(i["path"], i["expected"])
return
self.tests.append({'filepath': filepath, 'expected': expected})
def main(args):
parser = Options()
options, args = parser.parse_args()
if not options.html_manifest or not options.specialpowers or not options.host1 or not options.host2 or not options.signalling_server:
parser.print_usage()
return 2
package_options = get_package_options(parser, options)
if not package_options:
parser.print_usage()
return 2
if not os.path.isdir(options.specialpowers):
parser.error("SpecialPowers directory %s does not exist" % options.specialpowers)
return 2
if options.prefs and not os.path.isfile(options.prefs):
parser.error("Prefs file %s does not exist" % options.prefs)
return 2
if options.log_dest and not os.path.isdir(options.log_dest):
parser.error("Log directory %s does not exist" % options.log_dest)
return 2
log = mozlog.getLogger('steeplechase')
log.setLevel(mozlog.DEBUG)
if ':' in options.host1:
host, port = options.host1.split(':')
dm1 = DeviceManagerSUT(host, port)
else:
dm1 = DeviceManagerSUT(options.host1)
if ':' in options.host2:
host, port = options.host2.split(':')
dm2 = DeviceManagerSUT(host, port)
else:
dm2 = DeviceManagerSUT(options.host2)
remote_info = [{'dm': dm1,
'binary': package_options.binary,
'package': package_options.package,
'is_initiator': True,
'name': 'Client1'},
{'dm': dm2,
'binary': package_options.binary2,
'package': package_options.package2,
'is_initiator': False,
'name': 'Client2'}]
# first, push app
for info in remote_info:
dm = info['dm']
if info['binary']:
asset = Binary(path=info['binary'], log=log, dm=info['dm'], name=info['name'])
else:
asset = generate_package_asset(path=info['package'], log=log, dm=info['dm'], name=info['name'])
if options.setup:
asset.setup_test_root()
info['test_root'] = asset.test_root()
if options.setup:
log.info("Pushing app to %s...", info["name"])
asset.setup_client()
info['remote_app_path'] = asset.path_to_launch()
if not options.setup and not dm.fileExists(info['remote_app_path']):
log.error("App does not exist on %s, don't use --noSetup", info['name'])
return 2
pass_count, fail_count = 0, 0
if options.html_manifest:
manifest = TestManifest(strict=False)
manifest.read(options.html_manifest)
manifest_data = {"tests": [{"path": t["relpath"]} for t in manifest.active_tests(disabled=False, **mozinfo.info)]}
remote_port = 0
if options.remote_webserver:
result = re.search(':(\d+)', options.remote_webserver)
if result:
remote_port = int(result.groups()[0])
@json_response
def get_manifest(req):
return (200, manifest_data)
handlers = [{
'method': 'GET',
'path': '/manifest.json',
'function': get_manifest
}]
httpd = MozHttpd(host=moznetwork.get_ip(), port=remote_port, log_requests=True,
docroot=os.path.join(os.path.dirname(__file__), "..", "webharness"),
urlhandlers=handlers,
path_mappings={"/tests": os.path.dirname(options.html_manifest)})
httpd.start(block=False)
test = HTMLTests(httpd, remote_info, log, options)
html_pass_count, html_fail_count = test.run()
pass_count += html_pass_count
fail_count += html_fail_count
httpd.stop()
log.info("Result summary:")
log.info("Passed: %d" % pass_count)
log.info("Failed: %d" % fail_count)
return pass_count > 0 and fail_count == 0
def run_test(self, test):
if not self.httpd:
print "starting httpd"
self.start_httpd()
if not self.marionette:
self.start_marionette()
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if ((filename.startswith('test_') or filename.startswith('browser_')) and
(filename.endswith('.py') or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.run_test(filepath)
if self.marionette.check_for_crash():
return
return
mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1])
testloader = unittest.TestLoader()
suite = unittest.TestSuite()
if file_ext == '.ini':
testargs = {}
if self.type is not None:
testtypes = self.type.replace('+', ' +').replace('-', ' -').split()
for atype in testtypes:
if atype.startswith('+'):
testargs.update({ atype[1:]: 'true' })
elif atype.startswith('-'):
testargs.update({ atype[1:]: 'false' })
else:
testargs.update({ atype: 'true' })
manifest = TestManifest()
manifest.read(filepath)
all_tests = manifest.active_tests(disabled=False)
manifest_tests = manifest.active_tests(disabled=False,
device=self.device,
app=self.appName)
skip_tests = list(set([x['path'] for x in all_tests]) -
set([x['path'] for x in manifest_tests]))
for skipped in skip_tests:
self.logger.info('TEST-SKIP | %s | device=%s, app=%s' %
(os.path.basename(skipped),
self.device,
self.appName))
self.todo += 1
for i in manifest.get(tests=manifest_tests, **testargs):
self.run_test(i["path"])
if self.marionette.check_for_crash():
return
return
self.logger.info('TEST-START %s' % os.path.basename(test))
for handler in self.test_handlers:
if handler.match(os.path.basename(test)):
handler.add_tests_to_suite(mod_name,
filepath,
suite,
testloader,
self.marionette,
self.testvars,
**self.test_kwargs)
break
if suite.countTestCases():
runner = self.textrunnerclass(verbosity=3,
marionette=self.marionette)
results = runner.run(suite)
self.results.append(results)
self.failed += len(results.failures) + len(results.errors)
if hasattr(results, 'skipped'):
self.todo += len(results.skipped)
self.passed += results.passed
for failure in results.failures + results.errors:
self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL'))
if hasattr(results, 'unexpectedSuccesses'):
self.failed += len(results.unexpectedSuccesses)
for failure in results.unexpectedSuccesses:
self.failures.append((results.getInfo(failure), 'TEST-UNEXPECTED-PASS'))
if hasattr(results, 'expectedFailures'):
self.passed += len(results.expectedFailures)
def run_test_harness(options):
message_logger = MessageLogger(logger=None)
process_args = {'messageLogger': message_logger}
auto = RemoteAutomation(None, "fennec", processArgs=process_args)
if options is None:
raise ValueError("Invalid options specified, use --help for a list of valid options")
dm = options.dm
auto.setDeviceManager(dm)
mochitest = MochiRemote(auto, dm, options)
log = mochitest.log
message_logger.logger = log
mochitest.message_logger = message_logger
productPieces = options.remoteProductName.split('.')
if (productPieces is not None):
auto.setProduct(productPieces[0])
else:
auto.setProduct(options.remoteProductName)
auto.setAppName(options.remoteappname)
logParent = os.path.dirname(options.remoteLogFile)
dm.mkDir(logParent)
auto.setRemoteLog(options.remoteLogFile)
auto.setServerInfo(options.webServer, options.httpPort, options.sslPort)
mochitest.printDeviceInfo()
# Add Android version (SDK level) to mozinfo so that manifest entries
# can be conditional on android_version.
androidVersion = dm.shellCheckOutput(['getprop', 'ro.build.version.sdk'])
log.info(
"Android sdk version '%s'; will use this to filter manifests" %
str(androidVersion))
mozinfo.info['android_version'] = androidVersion
deviceRoot = dm.deviceRoot
if options.dmdPath:
dmdLibrary = "libdmd.so"
dmdPathOnDevice = os.path.join(deviceRoot, dmdLibrary)
dm.removeFile(dmdPathOnDevice)
dm.pushFile(os.path.join(options.dmdPath, dmdLibrary), dmdPathOnDevice)
options.dmdPath = deviceRoot
options.dumpOutputDirectory = deviceRoot
procName = options.app.split('/')[-1]
dm.killProcess(procName)
if options.robocopIni != "":
# turning buffering off as it's not used in robocop
message_logger.buffering = False
# sut may wait up to 300 s for a robocop am process before returning
dm.default_timeout = 320
if isinstance(options.manifestFile, TestManifest):
mp = options.manifestFile
else:
mp = TestManifest(strict=False)
mp.read(options.robocopIni)
filters = []
if options.totalChunks:
filters.append(
chunk_by_slice(options.thisChunk, options.totalChunks))
robocop_tests = mp.active_tests(exists=False, filters=filters, **mozinfo.info)
options.extraPrefs.append('browser.search.suggest.enabled=true')
options.extraPrefs.append('browser.search.suggest.prompted=true')
options.extraPrefs.append('layout.css.devPixelsPerPx=1.0')
options.extraPrefs.append('browser.chrome.dynamictoolbar=false')
options.extraPrefs.append('browser.snippets.enabled=false')
options.extraPrefs.append('browser.casting.enabled=true')
options.extraPrefs.append('extensions.autoupdate.enabled=false')
if (options.dm_trans == 'adb' and options.robocopApk):
dm._checkCmd(["install", "-r", options.robocopApk])
if not options.autorun:
# Force a single loop iteration. The iteration will start Fennec and
# the httpd server, but not actually run a test.
options.test_paths = [robocop_tests[0]['name']]
retVal = None
# Filtering tests
active_tests = []
for test in robocop_tests:
if options.test_paths and test['name'] not in options.test_paths:
continue
if 'disabled' in test:
log.info(
'TEST-INFO | skipping %s | %s' %
(test['name'], test['disabled']))
continue
active_tests.append(test)
log.suite_start([t['name'] for t in active_tests])
for test in active_tests:
# When running in a loop, we need to create a fresh profile for
# each cycle
if mochitest.localProfile:
options.profilePath = mochitest.localProfile
os.system("rm -Rf %s" % options.profilePath)
options.profilePath = None
mochitest.localProfile = options.profilePath
options.app = "am"
mochitest.nsprLogName = "nspr-%s.log" % test['name']
if options.autorun:
# This launches a test (using "am instrument") and instructs
# Fennec to /quit/ the browser (using Robocop:Quit) and to
# /finish/ all opened activities.
options.browserArgs = [
"instrument",
"-w",
"-e", "quit_and_finish", "1",
"-e", "deviceroot", deviceRoot,
"-e",
"class"]
options.browserArgs.append(
"org.mozilla.gecko.tests.%s" %
test['name'].split('.java')[0])
options.browserArgs.append(
"org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner")
else:
# This does not launch a test at all. It launches an activity
# that starts Fennec and then waits indefinitely, since cat
# never returns.
options.browserArgs = ["start",
"-n", "org.mozilla.roboexample.test/org.mozilla.gecko.LaunchFennecWithConfigurationActivity",
"&&", "cat"]
dm.default_timeout = sys.maxint # Forever.
mochitest.log.info("")
mochitest.log.info("Serving mochi.test Robocop root at http://%s:%s/tests/robocop/" %
(options.remoteWebServer, options.httpPort))
mochitest.log.info("")
# If the test is for checking the import from bookmarks then make
# sure there is data to import
if test['name'] == "testImportFromAndroid":
# Get the OS so we can run the insert in the apropriate
# database and following the correct table schema
osInfo = dm.getInfo("os")
devOS = " ".join(osInfo['os'])
if ("pandaboard" in devOS):
delete = [
'execsu',
'sqlite3',
"/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"]
else:
delete = [
'execsu',
'sqlite3',
"/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"]
if (options.dm_trans == "sut"):
dm._runCmds([{"cmd": " ".join(delete)}])
# Insert the bookmarks
log.info(
"Insert bookmarks in the default android browser database")
for i in range(20):
if ("pandaboard" in devOS):
cmd = [
'execsu',
'sqlite3',
"/data/data/com.android.browser/databases/browser2.db 'insert or replace into bookmarks(_id,title,url,folder,parent,position) values (" +
str(
30 +
i) +
",\"Bookmark" +
str(i) +
"\",\"http://www.bookmark" +
str(i) +
".com\",0,1," +
str(
100 +
i) +
");'"]
else:
cmd = [
'execsu',
'sqlite3',
"/data/data/com.android.browser/databases/browser.db 'insert into bookmarks(title,url,bookmark) values (\"Bookmark" +
str(i) +
"\",\"http://www.bookmark" +
str(i) +
".com\",1);'"]
if (options.dm_trans == "sut"):
dm._runCmds([{"cmd": " ".join(cmd)}])
try:
screenShotDir = "/mnt/sdcard/Robotium-Screenshots"
dm.removeDir(screenShotDir)
dm.recordLogcat()
result = mochitest.runTests(options)
if result != 0:
log.error("runTests() exited with code %s" % result)
log_result = mochitest.addLogData()
if result != 0 or log_result != 0:
mochitest.printDeviceInfo(printLogcat=True)
mochitest.printScreenshots(screenShotDir)
# Ensure earlier failures aren't overwritten by success on this
# run
if retVal is None or retVal == 0:
retVal = result
except:
log.error(
"Automation Error: Exception caught while running tests")
traceback.print_exc()
mochitest.stopServers()
try:
mochitest.cleanup(options)
except devicemanager.DMError:
# device error cleaning up... oh well!
pass
retVal = 1
break
finally:
# Clean-up added bookmarks
if test['name'] == "testImportFromAndroid":
if ("pandaboard" in devOS):
cmd_del = [
'execsu',
'sqlite3',
"/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"]
else:
cmd_del = [
'execsu',
'sqlite3',
"/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"]
if (options.dm_trans == "sut"):
dm._runCmds([{"cmd": " ".join(cmd_del)}])
if retVal is None:
log.warning("No tests run. Did you pass an invalid TEST_PATH?")
retVal = 1
else:
# if we didn't have some kind of error running the tests, make
# sure the tests actually passed
print "INFO | runtests.py | Test summary: start."
overallResult = mochitest.printLog()
print "INFO | runtests.py | Test summary: end."
if retVal == 0:
retVal = overallResult
else:
mochitest.nsprLogName = "nspr.log"
try:
dm.recordLogcat()
retVal = mochitest.runTests(options)
except:
log.error("Automation Error: Exception caught while running tests")
traceback.print_exc()
mochitest.stopServers()
try:
mochitest.cleanup(options)
except devicemanager.DMError:
# device error cleaning up... oh well!
pass
retVal = 1
mochitest.printDeviceInfo(printLogcat=True)
message_logger.finish()
return retVal
def run_python_tests(self,
tests=None,
test_objects=None,
subsuite=None,
verbose=False,
jobs=None,
python=None,
exitfirst=False,
extra=None,
**kwargs):
self._activate_test_virtualenvs(python)
if test_objects is None:
from moztest.resolve import TestResolver
resolver = self._spawn(TestResolver)
# If we were given test paths, try to find tests matching them.
test_objects = resolver.resolve_tests(paths=tests, flavor='python')
else:
# We've received test_objects from |mach test|. We need to ignore
# the subsuite because python-tests don't use this key like other
# harnesses do and |mach test| doesn't realize this.
subsuite = None
mp = TestManifest()
mp.tests.extend(test_objects)
filters = []
if subsuite == 'default':
filters.append(mpf.subsuite(None))
elif subsuite:
filters.append(mpf.subsuite(subsuite))
tests = mp.active_tests(filters=filters,
disabled=False,
python=self.virtualenv_manager.version_info[0],
**mozinfo.info)
if not tests:
submsg = "for subsuite '{}' ".format(subsuite) if subsuite else ""
message = "TEST-UNEXPECTED-FAIL | No tests collected " + \
"{}(Not in PYTHON_UNITTEST_MANIFESTS?)".format(submsg)
self.log(logging.WARN, 'python-test', {}, message)
return 1
parallel = []
sequential = []
os.environ.setdefault('PYTEST_ADDOPTS', '')
if extra:
os.environ['PYTEST_ADDOPTS'] += " " + " ".join(extra)
if exitfirst:
sequential = tests
os.environ['PYTEST_ADDOPTS'] += " -x"
else:
for test in tests:
if test.get('sequential'):
sequential.append(test)
else:
parallel.append(test)
self.jobs = jobs or cpu_count()
self.terminate = False
self.verbose = verbose
return_code = 0
def on_test_finished(result):
output, ret, test_path = result
for line in output:
self.log(logging.INFO, 'python-test', {'line': line.rstrip()},
'{line}')
if ret and not return_code:
self.log(logging.ERROR, 'python-test', {
'test_path': test_path,
'ret': ret
}, 'Setting retcode to {ret} from {test_path}')
return return_code or ret
with ThreadPoolExecutor(max_workers=self.jobs) as executor:
futures = [
executor.submit(self._run_python_test, test)
for test in parallel
]
try:
for future in as_completed(futures):
return_code = on_test_finished(future.result())
except KeyboardInterrupt:
# Hack to force stop currently running threads.
# https://gist.github.com/clchiou/f2608cbe54403edb0b13
executor._threads.clear()
thread._threads_queues.clear()
raise
for test in sequential:
return_code = on_test_finished(self._run_python_test(test))
if return_code and exitfirst:
break
self.log(logging.INFO, 'python-test', {'return_code': return_code},
'Return code from mach python-test: {return_code}')
return return_code
def run_test(self, test, expected='pass'):
if not self.httpd:
print "starting httpd"
self.start_httpd()
if not self.marionette:
self.start_marionette()
if self.emulator:
self.marionette.emulator.wait_for_homescreen(self.marionette)
# Retrieve capabilities for later use
if not self._capabilities:
self.capabilities
testargs = {}
if self.type is not None:
testtypes = self.type.replace('+', ' +').replace('-', ' -').split()
for atype in testtypes:
if atype.startswith('+'):
testargs.update({ atype[1:]: 'true' })
elif atype.startswith('-'):
testargs.update({ atype[1:]: 'false' })
else:
testargs.update({ atype: 'true' })
oop = testargs.get('oop', False)
if isinstance(oop, basestring):
oop = False if oop == 'false' else 'true'
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
if self.shuffle:
random.shuffle(files)
for filename in files:
if ((filename.startswith('test_') or filename.startswith('browser_')) and
(filename.endswith('.py') or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.run_test(filepath)
if self.marionette.check_for_crash():
return
return
mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1])
testloader = unittest.TestLoader()
suite = unittest.TestSuite()
if file_ext == '.ini':
manifest = TestManifest()
manifest.read(filepath)
manifest_tests = manifest.active_tests(exists=False,
disabled=True,
device=self.device,
app=self.appName,
**mozinfo.info)
unfiltered_tests = []
for test in manifest_tests:
if test.get('disabled'):
self.manifest_skipped_tests.append(test)
else:
unfiltered_tests.append(test)
target_tests = manifest.get(tests=unfiltered_tests, **testargs)
for test in unfiltered_tests:
if test['path'] not in [x['path'] for x in target_tests]:
test.setdefault('disabled', 'filtered by type (%s)' % self.type)
self.manifest_skipped_tests.append(test)
for test in self.manifest_skipped_tests:
self.logger.info('TEST-SKIP | %s | %s' % (
os.path.basename(test['path']),
test['disabled']))
self.todo += 1
if self.shuffle:
random.shuffle(target_tests)
for i in target_tests:
if not os.path.exists(i["path"]):
raise IOError("test file: %s does not exist" % i["path"])
self.run_test(i["path"], i["expected"])
if self.marionette.check_for_crash():
return
return
self.logger.info('TEST-START %s' % os.path.basename(test))
self.test_kwargs['expected'] = expected
self.test_kwargs['oop'] = oop
for handler in self.test_handlers:
if handler.match(os.path.basename(test)):
handler.add_tests_to_suite(mod_name,
filepath,
suite,
testloader,
self.marionette,
self.testvars,
**self.test_kwargs)
break
if suite.countTestCases():
runner = self.textrunnerclass(verbosity=3,
marionette=self.marionette,
capabilities=self.capabilities)
results = runner.run(suite)
self.results.append(results)
self.failed += len(results.failures) + len(results.errors)
if hasattr(results, 'skipped'):
self.todo += len(results.skipped)
self.passed += results.passed
for failure in results.failures + results.errors:
self.failures.append((results.getInfo(failure), failure.output, 'TEST-UNEXPECTED-FAIL'))
if hasattr(results, 'unexpectedSuccesses'):
self.failed += len(results.unexpectedSuccesses)
for failure in results.unexpectedSuccesses:
self.failures.append((results.getInfo(failure), 'TEST-UNEXPECTED-PASS'))
if hasattr(results, 'expectedFailures'):
self.passed += len(results.expectedFailures)
class ManifestParser():
test_logs = []
output = None
def __init__(self, manifest, git_link, git_branch):
# generate git path
self._test_href = git_link + 'tree/' + git_branch + '/tests/python/gaia-ui-tests/gaiatest/tests/'
# read manifest
self.manifest = TestManifest(manifests=(manifest,))
self._flame = self.manifest.active_tests(b2g=True, device='Flame')
# read tree herder
self._desktop = self.manifest.active_tests(b2g=True, device='desktop')
def _get_test(self, manifest, test_name):
for test in manifest:
if test_name in test['name']:
return test
def table_row(self, test):
name = test['name'].split('/')[-1]
suite_name = test['relpath'].split('/')[0]
app_name = os.path.basename(test['here'])
path_name = test['here'].split('gaiatest')[1]
link = self._test_href + test['relpath']
run_link = self._test_href + suite_name
class_link = self._test_href + os.path.dirname(test['relpath'])
flame = Device('Flame')
desktop = Device('desktop')
flame_test = self._get_test(self._flame, name)
desktop_test = self._get_test(self._desktop, name)
if flame_test:
if flame_test['expected'] == 'fail':
flame.status = 'XFailed'
flame.tooltip = 'Test is expected to fail'
elif 'disabled' in flame_test.keys():
flame.status = 'Disabled'
flame.tooltip = flame_test['disabled']
elif flame_test['expected'] == 'pass':
flame.status = 'Enabled'
flame.tooltip = 'Test is expected to pass'
if desktop_test:
if desktop_test.has_key('skip-if') and re.match('device\s*==\s*"desktop"', desktop_test['skip-if']):
pass
elif desktop_test['expected'] == 'fail':
desktop.status = 'XFailed'
desktop.tooltip = 'Test is expected to fail'
elif 'disabled' in desktop_test.keys():
desktop.status = 'Disabled'
desktop.tooltip = desktop_test['disabled']
elif desktop_test['expected'] == 'pass':
desktop.status = 'Enabled'
desktop.tooltip = 'Test is expected to pass'
self.test_logs.append(
html.tr([
html.td(
html.a(name, href_=link, target_='_blank'),
class_='col-name', title=path_name),
html.td(
html.a(suite_name, href_=run_link, target_='_blank'),
class_='col-run'),
html.td(
html.a(app_name, href_=class_link, target_='_blank'),
class_='col-class'),
html.td(flame.status, class_='col-flame ' + flame.status, title_=flame.tooltip),
html.td(desktop.status, class_='col-desktop ' + desktop.status,
title_=desktop.tooltip)
], class_='results-table-row')
)
def generate_html(self):
# generate table entry's
for test in self.manifest.tests:
self.table_row(test)
# main HTML file
doc = html.html(
html.head(
html.meta(charset='utf-8'),
html.title('Test Report'),
html.style(raw(pkg_resources.resource_string(
__name__, os.path.sep.join(['resources', 'style.css']))),
type='text/css')),
html.body(
html.script(raw(pkg_resources.resource_string(
__name__, os.path.sep.join(['resources', 'jquery.js']))),
type='text/javascript'),
html.script(raw(pkg_resources.resource_string(
__name__, os.path.sep.join(['resources', 'main.js']))),
type='text/javascript'),
html.h2('Test Status Mapping'),
html.br(),
html.p(
html.span('''The following table shows the functional Gaia UI tests and the targets they are currently run against.
Each test indicates the expected outcome from running the test.'''),
html.br(),
html.span('''Hover over the test name for the full path of the test file.
Hover over the expected state to see if there's an associated reason'''),
html.br(),
html.br(),
html.span('Generated on: %s' % time.strftime("%c"))
),
html.br(),
html.table([html.thead(
html.tr([
html.th('Test', rowspan_="2", colspan_="3"),
html.th('Device', colspan_="1"),
html.th('Desktop', colspan_="1")
]),
html.tr([
html.th('Flame'),
html.th('Desktop')
]),
html.tr([
html.th('Name', class_='sortable', col='name'),
html.th('Run', class_='sortable', col='run'),
html.th('Class', class_='sortable', col='class'),
html.th('State', class_='sortable', col='flame'),
html.th('State', class_='sortable', col='desktop'),
]), id='results-table-head'),
html.tbody(self.test_logs, id='results-table-body')], id='results-table')))
return doc.unicode()
def add_test(self, test, expected='pass', test_container=None):
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if (filename.startswith('test_') and
(filename.endswith('.py') or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.add_test(filepath)
return
testargs = {}
if self.type is not None:
testtypes = self.type.replace('+', ' +').replace('-', ' -').split()
for atype in testtypes:
if atype.startswith('+'):
testargs.update({ atype[1:]: 'true' })
elif atype.startswith('-'):
testargs.update({ atype[1:]: 'false' })
else:
testargs.update({ atype: 'true' })
testarg_b2g = bool(testargs.get('b2g'))
file_ext = os.path.splitext(os.path.split(filepath)[-1])[1]
if file_ext == '.ini':
manifest = TestManifest()
manifest.read(filepath)
filters = []
if self.test_tags:
filters.append(tags(self.test_tags))
manifest_tests = manifest.active_tests(exists=False,
disabled=True,
filters=filters,
device=self.device,
app=self.appName,
**mozinfo.info)
if len(manifest_tests) == 0:
self.logger.error("no tests to run using specified "
"combination of filters: {}".format(
manifest.fmt_filters()))
unfiltered_tests = []
for test in manifest_tests:
if test.get('disabled'):
self.manifest_skipped_tests.append(test)
else:
unfiltered_tests.append(test)
target_tests = manifest.get(tests=unfiltered_tests, **testargs)
for test in unfiltered_tests:
if test['path'] not in [x['path'] for x in target_tests]:
test.setdefault('disabled', 'filtered by type (%s)' % self.type)
self.manifest_skipped_tests.append(test)
for i in target_tests:
if not os.path.exists(i["path"]):
raise IOError("test file: %s does not exist" % i["path"])
file_ext = os.path.splitext(os.path.split(i['path'])[-1])[-1]
test_container = None
if i.get('test_container') and testarg_b2g:
if i.get('test_container') == "true":
test_container = True
elif i.get('test_container') == "false":
test_container = False
self.add_test(i["path"], i["expected"], test_container)
return
self.tests.append({'filepath': filepath, 'expected': expected, 'test_container': test_container})
class CLI(mozrunner.CLI):
"""Command line interface to mozmill."""
module = "mozmill"
def __init__(self, args):
# event handler plugin names
self.handlers = {}
for handler_class in handlers.handlers():
name = getattr(handler_class, 'name', handler_class.__name__)
self.handlers[name] = handler_class
self.jsbridge_port = jsbridge.find_port()
# add and parse options
mozrunner.CLI.__init__(self, args)
# Do not allow manifests and tests specified at the same time
if self.options.manifests and self.options.tests:
self.parser.error("Options %s and %s are mutually exclusive." %
(self.parser.get_option('-t'),
self.parser.get_option('-m')))
# read tests from manifests (if any)
self.manifest = TestManifest(manifests=self.options.manifests,
strict=False)
# expand user directory and check existence for the test
for test in self.options.tests:
testpath = os.path.expanduser(test)
realpath = os.path.realpath(testpath)
if not os.path.exists(testpath):
raise Exception("Not a valid test file/directory: %s" % test)
# collect the tests
def testname(t):
if os.path.isdir(realpath):
return os.path.join(test, os.path.relpath(t, testpath))
return test
tests = [{'name': testname(t), 'path': t}
for t in collect_tests(testpath)]
self.manifest.tests.extend(tests)
# list the tests and exit if specified
if self.options.list_tests:
for test in self.manifest.tests:
print test['path']
self.parser.exit()
# instantiate event handler plugins
self.event_handlers = []
for name, handler_class in self.handlers.items():
if name in self.options.disable:
continue
handler = handlers.instantiate_handler(handler_class, self.options)
if handler is not None:
self.event_handlers.append(handler)
for handler in self.options.handlers:
# user handlers
try:
handler_class = handlers.load_handler(handler)
except BaseException as e:
self.parser.error(str(e))
_handler = handlers.instantiate_handler(handler_class,
self.options)
if _handler is not None:
self.event_handlers.append(_handler)
# if in manual mode, ensure we're interactive
if self.options.manual:
self.options.interactive = True
def add_options(self, parser):
"""Add command line options."""
group = OptionGroup(parser, 'MozRunner options')
mozrunner.CLI.add_options(self, group)
parser.add_option_group(group)
group = OptionGroup(parser, 'MozMill options')
group.add_option("-t", "--test",
dest="tests",
default=[],
action='append',
help='Run test')
group.add_option("--timeout",
dest="timeout",
type="float",
default=JSBRIDGE_TIMEOUT,
help="Seconds before harness timeout if no "
"communication is taking place")
group.add_option("--restart",
dest='restart',
action='store_true',
default=False,
help="Restart the application and reset the "
"profile between each test file")
group.add_option("-m", "--manifest",
dest='manifests',
action='append',
metavar='MANIFEST',
help='test manifest .ini file')
group.add_option('-D', '--debug', dest="debug",
action="store_true",
default=False,
help="debug mode"
)
group.add_option('--list-tests',
dest='list_tests',
action='store_true',
default=False,
help="List test files that would be run, in order")
group.add_option('--handler',
dest='handlers',
action='append',
default=[],
metavar='PATH:CLASS',
help="Specify an event handler given a file PATH "
"and the CLASS in the file")
group.add_option('--screenshots-path',
dest='screenshots_path',
metavar='PATH',
help='Path of directory to use for screenshots')
if self.handlers:
group.add_option('--disable',
dest='disable',
action='append',
default=[],
metavar='HANDLER',
help="Disable a default event handler (%s)" %
','.join(self.handlers.keys()))
group.add_option('--manual', dest='manual',
action='store_true', default=False,
help="start the browser without running any tests")
parser.add_option_group(group)
# add option for included event handlers
for name, handler_class in self.handlers.items():
if hasattr(handler_class, 'add_options'):
group = OptionGroup(parser, '%s options' % name,
description=getattr(handler_class,
'__doc__', None))
handler_class.add_options(group)
parser.add_option_group(group)
def profile_args(self):
"""Setup profile settings for the profile object.
Returns arguments needed to make a profile object from
this command-line interface.
"""
profile_args = mozrunner.CLI.profile_args(self)
profile_args.setdefault('addons', []).extend(ADDONS)
profile_args['preferences'] = {
'extensions.jsbridge.port': self.jsbridge_port,
'focusmanager.testmode': True
}
if self.options.debug:
prefs = profile_args['preferences']
prefs['extensions.checkCompatibility'] = False
prefs['extensions.jsbridge.log'] = True
prefs['javascript.options.strict'] = True
return profile_args
def command_args(self):
"""Arguments to the application to be run."""
cmdargs = mozrunner.CLI.command_args(self)
if self.options.debug and '-jsconsole' not in cmdargs:
cmdargs.append('-jsconsole')
return cmdargs
def run(self):
"""CLI front end to run mozmill."""
# make sure you have tests to run
if (not self.manifest.tests) and (not self.options.manual):
self.parser.error("No tests found. Please specify with -t or -m")
# create a Mozrunner
runner = self.create_runner()
# create an instance of MozMill
mozmill = MozMill(runner, self.jsbridge_port,
jsbridge_timeout=self.options.timeout,
handlers=self.event_handlers,
screenshots_path=self.options.screenshots_path)
# set debugger arguments
mozmill.set_debugger(*self.debugger_arguments())
# load the mozmill + jsbridge extension but don't run any tests
# (for debugging)
if self.options.manual:
try:
mozmill.start_runner()
mozmill.runner.wait()
except (JSBridgeDisconnectError, KeyboardInterrupt):
pass
return
# run the tests
exception = None
tests = self.manifest.active_tests(**mozinfo.info)
try:
mozmill.run(tests, self.options.restart)
except:
exception_type, exception, tb = sys.exc_info()
# do whatever reporting you're going to do
results = mozmill.finish(fatal=exception is not None)
# exit on bad stuff happen
if exception:
traceback.print_exception(exception_type, exception, tb)
if exception or results.fails:
sys.exit(1)
# return results on success [currently unused]
return results
def add_test(self, test, expected='pass', oop=None):
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if ((filename.startswith('test_') or filename.startswith('browser_')) and
(filename.endswith('.py') or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.add_test(filepath)
return
testargs = {}
if self.type is not None:
testtypes = self.type.replace('+', ' +').replace('-', ' -').split()
for atype in testtypes:
if atype.startswith('+'):
testargs.update({ atype[1:]: 'true' })
elif atype.startswith('-'):
testargs.update({ atype[1:]: 'false' })
else:
testargs.update({ atype: 'true' })
# testarg_oop = either None, 'true' or 'false'.
testarg_oop = testargs.get('oop')
file_ext = os.path.splitext(os.path.split(filepath)[-1])[1]
if file_ext == '.ini':
manifest = TestManifest()
manifest.read(filepath)
manifest_tests = manifest.active_tests(exists=False,
disabled=True,
device=self.device,
app=self.appName,
**mozinfo.info)
unfiltered_tests = []
for test in manifest_tests:
if test.get('disabled'):
self.manifest_skipped_tests.append(test)
else:
unfiltered_tests.append(test)
# Don't filter tests with "oop" flag because manifest parser can't
# handle it well.
if testarg_oop is not None:
del testargs['oop']
target_tests = manifest.get(tests=unfiltered_tests, **testargs)
for test in unfiltered_tests:
if test['path'] not in [x['path'] for x in target_tests]:
test.setdefault('disabled', 'filtered by type (%s)' % self.type)
self.manifest_skipped_tests.append(test)
for test in self.manifest_skipped_tests:
self.logger.info('TEST-SKIP | %s | %s' % (
os.path.basename(test['path']),
test['disabled']))
self.todo += 1
for i in target_tests:
if not os.path.exists(i["path"]):
raise IOError("test file: %s does not exist" % i["path"])
# manifest_oop is either 'false', 'true' or 'both'. Anything
# else implies 'false'.
manifest_oop = i.get('oop', 'false')
# We only add an oop test when following conditions are met:
# 1) It's written by javascript because we have only
# MarionetteJSTestCase that supports oop mode.
# 2) we're running with "--type=+oop" or no "--type=-oop", which
# follows testarg_oop is either None or 'true' and must not
# be 'false'.
# 3) When no "--type=[+-]oop" is applied, all active tests are
# included in target_tests, so we must filter out those
# really capable of running in oop mode. Besides, oop tests
# must be explicitly specified for backward compatibility. So
# test manifest_oop equals to either 'both' or 'true'.
file_ext = os.path.splitext(os.path.split(i['path'])[-1])[-1]
if (file_ext == '.js' and
testarg_oop != 'false' and
(manifest_oop == 'both' or manifest_oop == 'true')):
self.add_test(i["path"], i["expected"], True)
# We only add an in-process test when following conditions are
# met:
# 1) we're running with "--type=-oop" or no "--type=+oop", which
# follows testarg_oop is either None or 'false' and must not
# be 'true'.
# 2) When no "--type=[+-]oop" is applied, all active tests are
# included in target_tests, so we must filter out those
# really capable of running in in-process mode.
if (testarg_oop != 'true' and
(manifest_oop == 'both' or manifest_oop != 'true')):
self.add_test(i["path"], i["expected"], False)
return
if oop is None:
# This test is added by directory enumeration or directly specified
# in argument list. We have no manifest information here so we just
# respect the "--type=[+-]oop" argument here.
oop = file_ext == '.js' and testarg_oop == 'true'
self.tests.append({'filepath': filepath, 'expected': expected, 'oop': oop})
def python_test(self,
tests=[],
test_objects=None,
subsuite=None,
verbose=False,
path_only=False,
stop=False,
jobs=1):
self._activate_virtualenv()
def find_tests_by_path():
import glob
files = []
for t in tests:
if t.endswith('.py') and os.path.isfile(t):
files.append(t)
elif os.path.isdir(t):
for root, _, _ in os.walk(t):
files += glob.glob(mozpath.join(root, 'test*.py'))
files += glob.glob(mozpath.join(root, 'unit*.py'))
else:
self.log(logging.WARN, 'python-test',
{'test': t},
'TEST-UNEXPECTED-FAIL | Invalid test: {test}')
if stop:
break
return files
# Python's unittest, and in particular discover, has problems with
# clashing namespaces when importing multiple test modules. What follows
# is a simple way to keep environments separate, at the price of
# launching Python multiple times. Most tests are run via mozunit,
# which produces output in the format Mozilla infrastructure expects.
# Some tests are run via pytest.
if test_objects is None:
# If we're not being called from `mach test`, do our own
# test resolution.
if path_only:
if tests:
test_objects = [{'path': p} for p in find_tests_by_path()]
else:
self.log(logging.WARN, 'python-test', {},
'TEST-UNEXPECTED-FAIL | No tests specified')
test_objects = []
else:
from mozbuild.testing import TestResolver
resolver = self._spawn(TestResolver)
if tests:
# If we were given test paths, try to find tests matching them.
test_objects = resolver.resolve_tests(paths=tests,
flavor='python')
else:
# Otherwise just run everything in PYTHON_UNITTEST_MANIFESTS
test_objects = resolver.resolve_tests(flavor='python')
if not test_objects:
message = 'TEST-UNEXPECTED-FAIL | No tests collected'
if not path_only:
message += ' (Not in PYTHON_UNITTEST_MANIFESTS? Try --path-only?)'
self.log(logging.WARN, 'python-test', {}, message)
return 1
mp = TestManifest()
mp.tests.extend(test_objects)
tests = mp.active_tests(disabled=False, **mozinfo.info)
self.jobs = jobs
self.terminate = False
self.verbose = verbose
return_code = 0
with ThreadPoolExecutor(max_workers=self.jobs) as executor:
futures = [executor.submit(self._run_python_test, test['path'])
for test in tests]
try:
for future in as_completed(futures):
output, ret, test_path = future.result()
for line in output:
self.log(logging.INFO, 'python-test', {'line': line.rstrip()}, '{line}')
if ret and not return_code:
self.log(logging.ERROR, 'python-test', {'test_path': test_path, 'ret': ret}, 'Setting retcode to {ret} from {test_path}')
return_code = return_code or ret
except KeyboardInterrupt:
# Hack to force stop currently running threads.
# https://gist.github.com/clchiou/f2608cbe54403edb0b13
executor._threads.clear()
thread._threads_queues.clear()
raise
self.log(logging.INFO, 'python-test', {'return_code': return_code}, 'Return code from mach python-test: {return_code}')
return return_code
def run_test(self, test, testtype):
if not self.httpd:
print "starting httpd"
self.start_httpd()
if not self.marionette:
self.start_marionette()
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if (filename.startswith("test_") or filename.startswith("browser_")) and (
filename.endswith(".py") or filename.endswith(".js")
):
filepath = os.path.join(root, filename)
self.run_test(filepath, testtype)
if self.marionette.check_for_crash():
return
return
mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1])
testloader = unittest.TestLoader()
suite = unittest.TestSuite()
if file_ext == ".ini":
testargs = {}
if testtype is not None:
testtypes = testtype.replace("+", " +").replace("-", " -").split()
for atype in testtypes:
if atype.startswith("+"):
testargs.update({atype[1:]: "true"})
elif atype.startswith("-"):
testargs.update({atype[1:]: "false"})
else:
testargs.update({atype: "true"})
manifest = TestManifest()
manifest.read(filepath)
if self.perf:
if self.perfserv is None:
self.perfserv = manifest.get("perfserv")[0]
machine_name = socket.gethostname()
try:
manifest.has_key("machine_name")
machine_name = manifest.get("machine_name")[0]
except:
self.logger.info("Using machine_name: %s" % machine_name)
os_name = platform.system()
os_version = platform.release()
self.perfrequest = datazilla.DatazillaRequest(
server=self.perfserv,
machine_name=machine_name,
os=os_name,
os_version=os_version,
platform=manifest.get("platform")[0],
build_name=manifest.get("build_name")[0],
version=manifest.get("version")[0],
revision=self.revision,
branch=manifest.get("branch")[0],
id=os.getenv("BUILD_ID"),
test_date=int(time.time()),
)
manifest_tests = manifest.active_tests(disabled=False)
for i in manifest.get(tests=manifest_tests, **testargs):
self.run_test(i["path"], testtype)
if self.marionette.check_for_crash():
return
return
self.logger.info("TEST-START %s" % os.path.basename(test))
for handler in self.test_handlers:
if handler.match(os.path.basename(test)):
handler.add_tests_to_suite(mod_name, filepath, suite, testloader, self.marionette, self.testvars)
break
if suite.countTestCases():
runner = MarionetteTextTestRunner(verbosity=3, perf=self.perf, marionette=self.marionette)
results = runner.run(suite)
self.results.append(results)
self.failed += len(results.failures) + len(results.errors)
if results.perfdata and options.perf:
self.perfrequest.add_datazilla_result(results.perfdata)
if hasattr(results, "skipped"):
self.todo += len(results.skipped) + len(results.expectedFailures)
self.passed += results.passed
for failure in results.failures + results.errors:
self.failures.append((results.getInfo(failure[0]), failure[1], "TEST-UNEXPECTED-FAIL"))
if hasattr(results, "unexpectedSuccess"):
self.failed += len(results.unexpectedSuccesses)
for failure in results.unexpectedSuccesses:
self.failures.append((results.getInfo(failure[0]), failure[1], "TEST-UNEXPECTED-PASS"))
