def test_manifest_subsuites(self):
"""
test subsuites and conditional subsuites
"""
relative_path = os.path.join(here, "subsuite.ini")
manifest = TestManifest(manifests=(relative_path,))
info = {"foo": "bar"}
# 6 tests total
tests = manifest.active_tests(exists=False, **info)
self.assertEqual(len(tests), 6)
# only 3 tests for subsuite bar when foo==bar
tests = manifest.active_tests(exists=False, filters=[subsuite("bar")], **info)
self.assertEqual(len(tests), 3)
# only 1 test for subsuite baz, regardless of conditions
other = {"something": "else"}
tests = manifest.active_tests(exists=False, filters=[subsuite("baz")], **info)
self.assertEqual(len(tests), 1)
tests = manifest.active_tests(exists=False, filters=[subsuite("baz")], **other)
self.assertEqual(len(tests), 1)
# 4 tests match when the condition doesn't match (all tests except
# the unconditional subsuite)
info = {"foo": "blah"}
tests = manifest.active_tests(exists=False, filters=[subsuite()], **info)
self.assertEqual(len(tests), 5)
# test for illegal subsuite value
manifest.tests[0]["subsuite"] = 'subsuite=bar,foo=="bar",type="nothing"'
with self.assertRaises(ParseError):
manifest.active_tests(exists=False, filters=[subsuite("foo")], **info)
def test_add_duplicates_to_set(self):
foo = lambda x, y: x
bar = lambda x, y: x
sub = subsuite('foo')
fl = filterlist([foo, bar, sub])
self.assertEquals(len(fl), 3)
self.assertEquals(fl[0], foo)
with self.assertRaises(ValueError):
fl.append(foo)
with self.assertRaises(ValueError):
fl.append(subsuite('bar'))
def test_subsuite(self):
sub1 = subsuite()
sub2 = subsuite('baz')
tests = deepcopy(self.tests)
tests = list(sub1(tests, {}))
self.assertNotIn(self.tests[4], tests)
self.assertEquals(tests[-1]['name'], 'test5')
tests = deepcopy(self.tests)
tests = list(sub2(tests, {}))
self.assertEquals(len(tests), 1)
self.assertIn(self.tests[4], tests)
def test_add_duplicates_to_list(self):
foo = lambda x, y: x
bar = lambda x, y: x
sub = subsuite('foo')
fl = filterlist([foo, bar, sub])
self.assertEquals(len(fl), 3)
self.assertEquals(fl[0], foo)
with self.assertRaises(ValueError):
fl.append(foo)
with self.assertRaises(ValueError):
fl.append(subsuite('bar'))
def test_subsuite(self):
sub1 = subsuite()
sub2 = subsuite('baz')
tests = deepcopy(self.tests)
tests = list(sub1(tests, {}))
self.assertNotIn(self.tests[5], tests)
self.assertEquals(len(tests), len(self.tests) - 1)
tests = deepcopy(self.tests)
tests = list(sub2(tests, {}))
self.assertEquals(len(tests), 1)
self.assertIn(self.tests[5], tests)
def test_subsuite(self):
sub1 = subsuite()
sub2 = subsuite('baz')
tests = deepcopy(self.tests)
tests = list(sub1(tests, {}))
self.assertNotIn(self.tests[5], tests)
self.assertEquals(len(tests), len(self.tests) - 1)
tests = deepcopy(self.tests)
tests = list(sub2(tests, {}))
self.assertEquals(len(tests), 1)
self.assertIn(self.tests[5], tests)
def test_subsuite(tests):
sub1 = subsuite()
sub2 = subsuite("baz")
ref = deepcopy(tests)
tests = list(sub1(tests, {}))
assert ref[5] not in tests
assert len(tests) == len(ref) - 1
tests = deepcopy(ref)
tests = list(sub2(tests, {}))
assert len(tests) == 1
assert ref[5] in tests
def test_add_duplicates_to_list():
foo = lambda x, y: x
bar = lambda x, y: x
sub = subsuite('foo')
fl = filterlist([foo, bar, sub])
assert len(fl) == 3
assert fl[0] == foo
with pytest.raises(ValueError):
fl.append(foo)
with pytest.raises(ValueError):
fl.append(subsuite('bar'))
def test_subsuite_condition(self):
sub1 = subsuite()
sub2 = subsuite('baz')
tests = deepcopy(self.tests)
tests = list(sub1(tests, {'foo': 'bar'}))
self.assertNotIn(self.tests[4], tests)
self.assertNotIn(self.tests[5], tests)
tests = deepcopy(self.tests)
tests = list(sub2(tests, {'foo': 'bar'}))
self.assertEquals(len(tests), 2)
self.assertEquals(tests[0]['name'], 'test4')
self.assertEquals(tests[1]['name'], 'test5')
def test_subsuite_condition(self):
sub1 = subsuite()
sub2 = subsuite('baz')
tests = deepcopy(self.tests)
tests = list(sub1(tests, {'foo': 'bar'}))
self.assertNotIn(self.tests[5], tests)
self.assertNotIn(self.tests[6], tests)
tests = deepcopy(self.tests)
tests = list(sub2(tests, {'foo': 'bar'}))
self.assertEquals(len(tests), 2)
self.assertEquals(tests[0]['name'], 'test5')
self.assertEquals(tests[1]['name'], 'test6')
def test_subsuite_condition(tests):
sub1 = subsuite()
sub2 = subsuite("baz")
ref = deepcopy(tests)
tests = list(sub1(tests, {"foo": "bar"}))
assert ref[5] not in tests
assert ref[6] not in tests
tests = deepcopy(ref)
tests = list(sub2(tests, {"foo": "bar"}))
assert len(tests) == 2
assert tests[0]["name"] == "test5"
assert tests[1]["name"] == "test6"
def test_subsuite_condition(tests):
sub1 = subsuite()
sub2 = subsuite('baz')
ref = deepcopy(tests)
tests = list(sub1(tests, {'foo': 'bar'}))
assert ref[5] not in tests
assert ref[6] not in tests
tests = deepcopy(ref)
tests = list(sub2(tests, {'foo': 'bar'}))
assert len(tests) == 2
assert tests[0]['name'] == 'test5'
assert tests[1]['name'] == 'test6'
def test_add_duplicates_to_list():
def foo(x, y):
return x
def bar(x, y):
return x
sub = subsuite("foo")
fl = filterlist([foo, bar, sub])
assert len(fl) == 3
assert fl[0] == foo
with pytest.raises(ValueError):
fl.append(foo)
with pytest.raises(ValueError):
fl.append(subsuite("bar"))
def test_manifest_subsuites(self):
"""
test subsuites and conditional subsuites
"""
relative_path = os.path.join(here, 'subsuite.ini')
manifest = TestManifest(manifests=(relative_path,))
info = {'foo': 'bar'}
# 6 tests total
tests = manifest.active_tests(exists=False, **info)
self.assertEquals(len(tests), 6)
# only 3 tests for subsuite bar when foo==bar
tests = manifest.active_tests(exists=False,
filters=[subsuite('bar')],
**info)
self.assertEquals(len(tests), 3)
# only 1 test for subsuite baz, regardless of conditions
other = {'something': 'else'}
tests = manifest.active_tests(exists=False,
filters=[subsuite('baz')],
**info)
self.assertEquals(len(tests), 1)
tests = manifest.active_tests(exists=False,
filters=[subsuite('baz')],
**other)
self.assertEquals(len(tests), 1)
# 4 tests match when the condition doesn't match (all tests except
# the unconditional subsuite)
info = {'foo': 'blah'}
tests = manifest.active_tests(exists=False,
filters=[subsuite()],
**info)
self.assertEquals(len(tests), 5)
# test for illegal subsuite value
manifest.tests[0]['subsuite'] = 'subsuite=bar,foo=="bar",type="nothing"'
with self.assertRaises(ParseError):
manifest.active_tests(exists=False,
filters=[subsuite('foo')],
**info)
def run_python_tests(
command_context,
tests=None,
test_objects=None,
subsuite=None,
verbose=False,
jobs=None,
exitfirst=False,
extra=None,
**kwargs
):
command_context.activate_virtualenv()
if test_objects is None:
from moztest.resolve import TestResolver
resolver = command_context._spawn(TestResolver)
# If we were given test paths, try to find tests matching them.
test_objects = resolver.resolve_tests(paths=tests, flavor="python")
else:
# We've received test_objects from |mach test|. We need to ignore
# the subsuite because python-tests don't use this key like other
# harnesses do and |mach test| doesn't realize this.
subsuite = None
mp = TestManifest()
mp.tests.extend(test_objects)
filters = []
if subsuite == "default":
filters.append(mpf.subsuite(None))
elif subsuite:
filters.append(mpf.subsuite(subsuite))
tests = mp.active_tests(
filters=filters,
disabled=False,
python=command_context.virtualenv_manager.version_info()[0],
**mozinfo.info
)
if not tests:
submsg = "for subsuite '{}' ".format(subsuite) if subsuite else ""
message = (
"TEST-UNEXPECTED-FAIL | No tests collected "
+ "{}(Not in PYTHON_UNITTEST_MANIFESTS?)".format(submsg)
)
command_context.log(logging.WARN, "python-test", {}, message)
return 1
parallel = []
sequential = []
os.environ.setdefault("PYTEST_ADDOPTS", "")
if extra:
os.environ["PYTEST_ADDOPTS"] += " " + " ".join(extra)
installed_requirements = set()
for test in tests:
if (
test.get("requirements")
and test["requirements"] not in installed_requirements
):
command_context.virtualenv_manager.install_pip_requirements(
test["requirements"], quiet=True
)
installed_requirements.add(test["requirements"])
if exitfirst:
sequential = tests
os.environ["PYTEST_ADDOPTS"] += " -x"
else:
for test in tests:
if test.get("sequential"):
sequential.append(test)
else:
parallel.append(test)
jobs = jobs or cpu_count()
return_code = 0
def on_test_finished(result):
output, ret, test_path = result
for line in output:
command_context.log(
logging.INFO, "python-test", {"line": line.rstrip()}, "{line}"
)
if ret and not return_code:
command_context.log(
logging.ERROR,
"python-test",
{"test_path": test_path, "ret": ret},
"Setting retcode to {ret} from {test_path}",
)
return return_code or ret
with ThreadPoolExecutor(max_workers=jobs) as executor:
futures = [
executor.submit(_run_python_test, command_context, test, jobs, verbose)
for test in parallel
]
try:
for future in as_completed(futures):
return_code = on_test_finished(future.result())
except KeyboardInterrupt:
# Hack to force stop currently running threads.
# https://gist.github.com/clchiou/f2608cbe54403edb0b13
executor._threads.clear()
thread._threads_queues.clear()
raise
for test in sequential:
return_code = on_test_finished(
_run_python_test(command_context, test, jobs, verbose)
)
if return_code and exitfirst:
break
command_context.log(
logging.INFO,
"python-test",
{"return_code": return_code},
"Return code from mach python-test: {return_code}",
)
return return_code
def run_python_tests(self,
tests=[],
test_objects=None,
subsuite=None,
verbose=False,
stop=False,
jobs=1):
self._activate_virtualenv()
def find_tests_by_path():
import glob
files = []
for t in tests:
if t.endswith('.py') and os.path.isfile(t):
files.append(t)
elif os.path.isdir(t):
for root, _, _ in os.walk(t):
files += glob.glob(mozpath.join(root, 'test*.py'))
files += glob.glob(mozpath.join(root, 'unit*.py'))
else:
self.log(logging.WARN, 'python-test', {'test': t},
'TEST-UNEXPECTED-FAIL | Invalid test: {test}')
if stop:
break
return files
# Python's unittest, and in particular discover, has problems with
# clashing namespaces when importing multiple test modules. What follows
# is a simple way to keep environments separate, at the price of
# launching Python multiple times. Most tests are run via mozunit,
# which produces output in the format Mozilla infrastructure expects.
# Some tests are run via pytest.
if test_objects is None:
from moztest.resolve import TestResolver
resolver = self._spawn(TestResolver)
if tests:
# If we were given test paths, try to find tests matching them.
test_objects = resolver.resolve_tests(paths=tests,
flavor='python')
else:
# Otherwise just run everything in PYTHON_UNITTEST_MANIFESTS
test_objects = resolver.resolve_tests(flavor='python')
mp = TestManifest()
mp.tests.extend(test_objects)
filters = []
if subsuite == 'default':
filters.append(mpf.subsuite(None))
elif subsuite:
filters.append(mpf.subsuite(subsuite))
tests = mp.active_tests(filters=filters,
disabled=False,
**mozinfo.info)
if not tests:
submsg = "for subsuite '{}' ".format(subsuite) if subsuite else ""
message = "TEST-UNEXPECTED-FAIL | No tests collected " + \
"{}(Not in PYTHON_UNITTEST_MANIFESTS?)".format(submsg)
self.log(logging.WARN, 'python-test', {}, message)
return 1
parallel = []
sequential = []
for test in tests:
if test.get('sequential'):
sequential.append(test)
else:
parallel.append(test)
self.jobs = jobs
self.terminate = False
self.verbose = verbose
return_code = 0
def on_test_finished(result):
output, ret, test_path = result
for line in output:
self.log(logging.INFO, 'python-test', {'line': line.rstrip()},
'{line}')
if ret and not return_code:
self.log(logging.ERROR, 'python-test', {
'test_path': test_path,
'ret': ret
}, 'Setting retcode to {ret} from {test_path}')
return return_code or ret
with ThreadPoolExecutor(max_workers=self.jobs) as executor:
futures = [
executor.submit(self._run_python_test, test['path'])
for test in parallel
]
try:
for future in as_completed(futures):
return_code = on_test_finished(future.result())
except KeyboardInterrupt:
# Hack to force stop currently running threads.
# https://gist.github.com/clchiou/f2608cbe54403edb0b13
executor._threads.clear()
thread._threads_queues.clear()
raise
for test in sequential:
return_code = on_test_finished(self._run_python_test(test['path']))
self.log(logging.INFO, 'python-test', {'return_code': return_code},
'Return code from mach python-test: {return_code}')
return return_code
def run_python_tests(self,
tests=None,
test_objects=None,
subsuite=None,
verbose=False,
jobs=None,
python=None,
exitfirst=False,
extra=None,
**kwargs):
self._activate_test_virtualenvs(python)
if test_objects is None:
from moztest.resolve import TestResolver
resolver = self._spawn(TestResolver)
# If we were given test paths, try to find tests matching them.
test_objects = resolver.resolve_tests(paths=tests, flavor='python')
else:
# We've received test_objects from |mach test|. We need to ignore
# the subsuite because python-tests don't use this key like other
# harnesses do and |mach test| doesn't realize this.
subsuite = None
mp = TestManifest()
mp.tests.extend(test_objects)
filters = []
if subsuite == 'default':
filters.append(mpf.subsuite(None))
elif subsuite:
filters.append(mpf.subsuite(subsuite))
tests = mp.active_tests(filters=filters,
disabled=False,
python=self.virtualenv_manager.version_info[0],
**mozinfo.info)
if not tests:
submsg = "for subsuite '{}' ".format(subsuite) if subsuite else ""
message = "TEST-UNEXPECTED-FAIL | No tests collected " + \
"{}(Not in PYTHON_UNITTEST_MANIFESTS?)".format(submsg)
self.log(logging.WARN, 'python-test', {}, message)
return 1
parallel = []
sequential = []
os.environ.setdefault('PYTEST_ADDOPTS', '')
if extra:
os.environ['PYTEST_ADDOPTS'] += " " + " ".join(extra)
if exitfirst:
sequential = tests
os.environ['PYTEST_ADDOPTS'] += " -x"
else:
for test in tests:
if test.get('sequential'):
sequential.append(test)
else:
parallel.append(test)
self.jobs = jobs or cpu_count()
self.terminate = False
self.verbose = verbose
return_code = 0
def on_test_finished(result):
output, ret, test_path = result
for line in output:
self.log(logging.INFO, 'python-test', {'line': line.rstrip()},
'{line}')
if ret and not return_code:
self.log(logging.ERROR, 'python-test', {
'test_path': test_path,
'ret': ret
}, 'Setting retcode to {ret} from {test_path}')
return return_code or ret
with ThreadPoolExecutor(max_workers=self.jobs) as executor:
futures = [
executor.submit(self._run_python_test, test)
for test in parallel
]
try:
for future in as_completed(futures):
return_code = on_test_finished(future.result())
except KeyboardInterrupt:
# Hack to force stop currently running threads.
# https://gist.github.com/clchiou/f2608cbe54403edb0b13
executor._threads.clear()
thread._threads_queues.clear()
raise
for test in sequential:
return_code = on_test_finished(self._run_python_test(test))
if return_code and exitfirst:
break
self.log(logging.INFO, 'python-test', {'return_code': return_code},
'Return code from mach python-test: {return_code}')
return return_code
def run_python_tests(self,
tests=[],
test_objects=None,
subsuite=None,
verbose=False,
stop=False,
jobs=1):
self._activate_virtualenv()
def find_tests_by_path():
import glob
files = []
for t in tests:
if t.endswith('.py') and os.path.isfile(t):
files.append(t)
elif os.path.isdir(t):
for root, _, _ in os.walk(t):
files += glob.glob(mozpath.join(root, 'test*.py'))
files += glob.glob(mozpath.join(root, 'unit*.py'))
else:
self.log(logging.WARN, 'python-test',
{'test': t},
'TEST-UNEXPECTED-FAIL | Invalid test: {test}')
if stop:
break
return files
# Python's unittest, and in particular discover, has problems with
# clashing namespaces when importing multiple test modules. What follows
# is a simple way to keep environments separate, at the price of
# launching Python multiple times. Most tests are run via mozunit,
# which produces output in the format Mozilla infrastructure expects.
# Some tests are run via pytest.
if test_objects is None:
from mozbuild.testing import TestResolver
resolver = self._spawn(TestResolver)
if tests:
# If we were given test paths, try to find tests matching them.
test_objects = resolver.resolve_tests(paths=tests,
flavor='python')
else:
# Otherwise just run everything in PYTHON_UNITTEST_MANIFESTS
test_objects = resolver.resolve_tests(flavor='python')
mp = TestManifest()
mp.tests.extend(test_objects)
if not mp.tests:
message = 'TEST-UNEXPECTED-FAIL | No tests collected ' + \
'(Not in PYTHON_UNITTEST_MANIFESTS?)'
self.log(logging.WARN, 'python-test', {}, message)
return 1
filters = []
if subsuite == 'default':
filters.append(mpf.subsuite(None))
elif subsuite:
filters.append(mpf.subsuite(subsuite))
tests = mp.active_tests(filters=filters, disabled=False, **mozinfo.info)
parallel = []
sequential = []
for test in tests:
if test.get('sequential'):
sequential.append(test)
else:
parallel.append(test)
self.jobs = jobs
self.terminate = False
self.verbose = verbose
return_code = 0
def on_test_finished(result):
output, ret, test_path = result
for line in output:
self.log(logging.INFO, 'python-test', {'line': line.rstrip()}, '{line}')
if ret and not return_code:
self.log(logging.ERROR, 'python-test', {'test_path': test_path, 'ret': ret},
'Setting retcode to {ret} from {test_path}')
return return_code or ret
with ThreadPoolExecutor(max_workers=self.jobs) as executor:
futures = [executor.submit(self._run_python_test, test['path'])
for test in parallel]
try:
for future in as_completed(futures):
return_code = on_test_finished(future.result())
except KeyboardInterrupt:
# Hack to force stop currently running threads.
# https://gist.github.com/clchiou/f2608cbe54403edb0b13
executor._threads.clear()
thread._threads_queues.clear()
raise
for test in sequential:
return_code = on_test_finished(self._run_python_test(test['path']))
self.log(logging.INFO, 'python-test', {'return_code': return_code},
'Return code from mach python-test: {return_code}')
return return_code