def resolve_tests_by_suite(paths):
resolver = TestResolver.from_environment(cwd=here)
_, run_tests = resolver.resolve_metadata(paths)
suite_to_tests = defaultdict(list)
# A dictionary containing all the input paths that we haven't yet
# assigned to a specific test flavor.
remaining_paths_by_suite = defaultdict(lambda: set(paths))
for test in run_tests:
key, _ = get_suite_definition(test['flavor'], test.get('subsuite'), strict=True)
test_path = test.get('srcdir_relpath')
if test_path is None:
continue
found_path = None
for path in remaining_paths_by_suite[key]:
if test_path.startswith(path):
found_path = path
break
if found_path:
suite_to_tests[key].append(found_path)
remaining_paths_by_suite[key].remove(found_path)
return suite_to_tests
def filter_tasks_by_paths(tasks, paths):
resolver = TestResolver.from_environment(cwd=here)
run_suites, run_tests = resolver.resolve_metadata(paths)
flavors = set([(t['flavor'], t.get('subsuite')) for t in run_tests])
task_regexes = set()
for flavor, subsuite in flavors:
_, suite = get_suite_definition(flavor, subsuite, strict=True)
if 'task_regex' not in suite:
print("warning: no tasks could be resolved from flavor '{}'{}".format(
flavor, " and subsuite '{}'".format(subsuite) if subsuite else ""))
continue
task_regexes.update(suite['task_regex'])
def match_task(task):
return any(re.search(pattern, task) for pattern in task_regexes)
return filter(match_task, tasks)
def resolve_tests_by_suite(paths):
resolver = TestResolver.from_environment(cwd=here)
_, run_tests = resolver.resolve_metadata(paths)
suite_to_tests = defaultdict(list)
for test in run_tests:
suite = get_suite_definition(test['flavor'], test.get('subsuite'), strict=True)
if 'mozharness_name' in suite:
key = suite['mozharness_name']
else:
key = test['flavor']
subsuite = test.get('subsuite')
if subsuite:
key += '-' + subsuite
suite_to_tests[key].append(test['srcdir_relpath'])
return suite_to_tests
def test(self, what, extra_args, **log_args):
"""Run tests from names or paths.
mach test accepts arguments specifying which tests to run. Each argument
can be:
* The path to a test file
* A directory containing tests
* A test suite name
* An alias to a test suite name (codes used on TreeHerder)
If no input is provided, tests will be run based on files changed in
the local tree. Relevant tests, tags, or flavors are determined by
IMPACTED_TESTS annotations in moz.build files relevant to the
changed files.
When paths or directories are given, they are first resolved to test
files known to the build system.
If resolved tests belong to more than one test type/flavor/harness,
the harness for each relevant type/flavor will be invoked. e.g. if
you specify a directory with xpcshell and browser chrome mochitests,
both harnesses will be invoked.
"""
from mozlog.commandline import setup_logging
from mozlog.handlers import StreamHandler
from moztest.resolve import get_suite_definition, TestResolver, TEST_SUITES
resolver = self._spawn(TestResolver)
run_suites, run_tests = resolver.resolve_metadata(what)
if not run_suites and not run_tests:
print(UNKNOWN_TEST)
return 1
if log_args.get('debugger', None):
import mozdebug
if not mozdebug.get_debugger_info(log_args.get('debugger')):
sys.exit(1)
extra_args_debugger_notation = '='.join(
['--debugger', log_args.get('debugger')]).encode('ascii')
if extra_args:
extra_args.append(extra_args_debugger_notation)
else:
extra_args = [extra_args_debugger_notation]
# Create shared logger
format_args = {'level': self._mach_context.settings['test']['level']}
if not run_suites and len(run_tests) == 1:
format_args['verbose'] = True
format_args['compact'] = False
default_format = self._mach_context.settings['test']['format']
log = setup_logging('mach-test', log_args,
{default_format: sys.stdout}, format_args)
for handler in log.handlers:
if isinstance(handler, StreamHandler):
handler.formatter.inner.summary_on_shutdown = True
status = None
for suite_name in run_suites:
suite = TEST_SUITES[suite_name]
kwargs = suite['kwargs']
kwargs['log'] = log
if 'mach_command' in suite:
res = self._mach_context.commands.dispatch(
suite['mach_command'],
self._mach_context,
argv=extra_args,
**kwargs)
if res:
status = res
buckets = {}
for test in run_tests:
key = (test['flavor'], test.get('subsuite', ''))
buckets.setdefault(key, []).append(test)
for (flavor, subsuite), tests in sorted(buckets.items()):
_, m = get_suite_definition(flavor, subsuite)
if 'mach_command' not in m:
substr = '-{}'.format(subsuite) if subsuite else ''
print(UNKNOWN_FLAVOR % (flavor, substr))
status = 1
continue
kwargs = dict(m['kwargs'])
kwargs['log'] = log
res = self._mach_context.commands.dispatch(m['mach_command'],
self._mach_context,
argv=extra_args,
test_objects=tests,
**kwargs)
if res:
status = res
log.shutdown()
return status
def run_mochitest_general(self,
flavor=None,
test_objects=None,
resolve_tests=True,
**kwargs):
from mochitest_options import ALL_FLAVORS
from mozlog.commandline import setup_logging
from mozlog.handlers import StreamHandler
from moztest.resolve import get_suite_definition
buildapp = None
for app in SUPPORTED_APPS:
if is_buildapp_in(app)(self):
buildapp = app
break
flavors = None
if flavor:
for fname, fobj in ALL_FLAVORS.iteritems():
if flavor in fobj['aliases']:
if buildapp not in fobj['enabled_apps']:
continue
flavors = [fname]
break
else:
flavors = [
f for f, v in ALL_FLAVORS.iteritems()
if buildapp in v['enabled_apps']
]
from mozbuild.controller.building import BuildDriver
self._ensure_state_subdir_exists('.')
test_paths = kwargs['test_paths']
kwargs['test_paths'] = []
mochitest = self._spawn(MochitestRunner)
tests = []
if resolve_tests:
tests = mochitest.resolve_tests(test_paths,
test_objects,
cwd=self._mach_context.cwd)
if not kwargs.get('log'):
# Create shared logger
format_args = {
'level': self._mach_context.settings['test']['level']
}
if len(tests) == 1:
format_args['verbose'] = True
format_args['compact'] = False
default_format = self._mach_context.settings['test']['format']
kwargs['log'] = setup_logging('mach-mochitest', kwargs,
{default_format: sys.stdout},
format_args)
for handler in kwargs['log'].handlers:
if isinstance(handler, StreamHandler):
handler.formatter.inner.summary_on_shutdown = True
driver = self._spawn(BuildDriver)
driver.install_tests(tests)
subsuite = kwargs.get('subsuite')
if subsuite == 'default':
kwargs['subsuite'] = None
suites = defaultdict(list)
unsupported = set()
for test in tests:
# Filter out non-mochitests and unsupported flavors.
if test['flavor'] not in ALL_FLAVORS:
continue
key = (test['flavor'], test.get('subsuite', ''))
if test['flavor'] not in flavors:
unsupported.add(key)
continue
if subsuite == 'default':
# "--subsuite default" means only run tests that don't have a subsuite
if test.get('subsuite'):
unsupported.add(key)
continue
elif subsuite and test.get('subsuite', '') != subsuite:
unsupported.add(key)
continue
suites[key].append(test)
if ('mochitest', 'media') in suites:
req = os.path.join('testing', 'tools', 'websocketprocessbridge',
'websocketprocessbridge_requirements.txt')
self.virtualenv_manager.activate()
self.virtualenv_manager.install_pip_requirements(
req, require_hashes=False)
# sys.executable is used to start the websocketprocessbridge, though for some
# reason it doesn't get set when calling `activate_this.py` in the virtualenv.
sys.executable = self.virtualenv_manager.python_path
# This is a hack to introduce an option in mach to not send
# filtered tests to the mochitest harness. Mochitest harness will read
# the master manifest in that case.
if not resolve_tests:
for flavor in flavors:
key = (flavor, kwargs.get('subsuite'))
suites[key] = []
if not suites:
# Make it very clear why no tests were found
if not unsupported:
print(
TESTS_NOT_FOUND.format('\n'.join(
sorted(list(test_paths or test_objects)))))
return 1
msg = []
for f, s in unsupported:
fobj = ALL_FLAVORS[f]
apps = fobj['enabled_apps']
name = fobj['aliases'][0]
if s:
name = '{} --subsuite {}'.format(name, s)
if buildapp not in apps:
reason = 'requires {}'.format(' or '.join(apps))
else:
reason = 'excluded by the command line'
msg.append(' mochitest -f {} ({})'.format(name, reason))
print(
SUPPORTED_TESTS_NOT_FOUND.format(buildapp,
'\n'.join(sorted(msg))))
return 1
if buildapp == 'android':
from mozrunner.devices.android_device import grant_runtime_permissions
from mozrunner.devices.android_device import verify_android_device
app = kwargs.get('app')
if not app:
app = self.substs["ANDROID_PACKAGE_NAME"]
device_serial = kwargs.get('deviceSerial')
# verify installation
verify_android_device(self,
install=True,
xre=False,
network=True,
app=app,
device_serial=device_serial)
grant_runtime_permissions(self, app, device_serial=device_serial)
run_mochitest = mochitest.run_android_test
else:
run_mochitest = mochitest.run_desktop_test
overall = None
for (flavor, subsuite), tests in sorted(suites.items()):
_, suite = get_suite_definition(flavor, subsuite)
if 'test_paths' in suite['kwargs']:
del suite['kwargs']['test_paths']
harness_args = kwargs.copy()
harness_args.update(suite['kwargs'])
result = run_mochitest(self._mach_context,
tests=tests,
**harness_args)
if result:
overall = result
# Halt tests on keyboard interrupt
if result == -1:
break
# Only shutdown the logger if we created it
if kwargs['log'].name == 'mach-mochitest':
kwargs['log'].shutdown()
return overall
def test(self, what, extra_args, **log_args):
"""Run tests from names or paths.
mach test accepts arguments specifying which tests to run. Each argument
can be:
* The path to a test file
* A directory containing tests
* A test suite name
* An alias to a test suite name (codes used on TreeHerder)
When paths or directories are given, they are first resolved to test
files known to the build system.
If resolved tests belong to more than one test type/flavor/harness,
the harness for each relevant type/flavor will be invoked. e.g. if
you specify a directory with xpcshell and browser chrome mochitests,
both harnesses will be invoked.
Warning: `mach test` does not automatically re-build.
Please remember to run `mach build` when necessary.
EXAMPLES
Run all test files in the devtools/client/shared/redux/middleware/xpcshell/
directory:
`./mach test devtools/client/shared/redux/middleware/xpcshell/`
The below command prints a short summary of results instead of
the default more verbose output.
Do not forget the - (minus sign) after --log-grouped!
`./mach test --log-grouped - devtools/client/shared/redux/middleware/xpcshell/`
"""
from mozlog.commandline import setup_logging
from mozlog.handlers import StreamHandler
from moztest.resolve import get_suite_definition, TestResolver, TEST_SUITES
resolver = self._spawn(TestResolver)
run_suites, run_tests = resolver.resolve_metadata(what)
if not run_suites and not run_tests:
print(UNKNOWN_TEST)
return 1
if log_args.get("debugger", None):
import mozdebug
if not mozdebug.get_debugger_info(log_args.get("debugger")):
sys.exit(1)
extra_args_debugger_notation = "=".join(
["--debugger", log_args.get("debugger")]).encode("ascii")
if extra_args:
extra_args.append(extra_args_debugger_notation)
else:
extra_args = [extra_args_debugger_notation]
# Create shared logger
format_args = {"level": self._mach_context.settings["test"]["level"]}
if not run_suites and len(run_tests) == 1:
format_args["verbose"] = True
format_args["compact"] = False
default_format = self._mach_context.settings["test"]["format"]
log = setup_logging("mach-test", log_args,
{default_format: sys.stdout}, format_args)
for handler in log.handlers:
if isinstance(handler, StreamHandler):
handler.formatter.inner.summary_on_shutdown = True
status = None
for suite_name in run_suites:
suite = TEST_SUITES[suite_name]
kwargs = suite["kwargs"]
kwargs["log"] = log
kwargs.setdefault("subsuite", None)
if "mach_command" in suite:
res = self._mach_context.commands.dispatch(
suite["mach_command"],
self._mach_context,
argv=extra_args,
**kwargs)
if res:
status = res
buckets = {}
for test in run_tests:
key = (test["flavor"], test.get("subsuite", ""))
buckets.setdefault(key, []).append(test)
for (flavor, subsuite), tests in sorted(buckets.items()):
_, m = get_suite_definition(flavor, subsuite)
if "mach_command" not in m:
substr = "-{}".format(subsuite) if subsuite else ""
print(UNKNOWN_FLAVOR % (flavor, substr))
status = 1
continue
kwargs = dict(m["kwargs"])
kwargs["log"] = log
kwargs.setdefault("subsuite", None)
res = self._mach_context.commands.dispatch(m["mach_command"],
self._mach_context,
argv=extra_args,
test_objects=tests,
**kwargs)
if res:
status = res
log.shutdown()
return status
def chunk_manifests(flavor, subsuite, platform, chunks, manifests):
"""Run the chunking algorithm.
Args:
platform (str): Platform used to find runtime info.
chunks (int): Number of chunks to split manifests into.
manifests(list): Manifests to chunk.
Returns:
A list of length `chunks` where each item contains a list of manifests
that run in that chunk.
"""
# Obtain the suite definition given the flavor and subsuite which often
# do not perfectly map onto the actual suite name in taskgraph.
# This value will be used to retrive runtime information for that suite.
suite_name, _ = get_suite_definition(flavor, subsuite)
runtimes = get_runtimes(platform, suite_name)
if flavor != "web-platform-tests":
return [
c[1] for c in chunk_by_runtime(
None, chunks, runtimes).get_chunked_manifests(manifests)
]
paths = {k: v for k, v in wpt_group_translation.items() if k in manifests}
# Python2 does not support native dictionary sorting, so use an OrderedDict
# instead, appending in order of highest to lowest runtime.
runtimes = OrderedDict(
sorted(runtimes.items(), key=lambda x: x[1], reverse=True))
# Keep track of test paths for each chunk, and the runtime information.
chunked_manifests = [[[], 0] for _ in range(chunks)]
# Begin chunking the test paths in order from highest running time to lowest.
# The keys of runtimes dictionary should match the keys of the test paths
# dictionary.
for key, rt in runtimes.items():
# Sort the chunks from fastest to slowest, based on runtime info
# at x[1], then the number of test paths.
chunked_manifests.sort(key=lambda x: (x[1], len(x[0])))
# Look up if there are any test paths under the key in the paths dict.
test_paths = paths[key]
if test_paths:
# Add the full set of paths that live under the key and increase the
# total runtime counter by the value reported in runtimes.
chunked_manifests[0][0].extend(test_paths)
# chunked_manifests[0][0].append(key)
chunked_manifests[0][1] += rt
# Remove the key and test_paths since it has been scheduled.
paths.pop(key)
# Same goes for the value in runtimes dict.
runtimes.pop(key)
# Sort again prior to the next step.
chunked_manifests.sort(key=lambda x: (x[1], len(x[0])))
# Spread out the remaining test paths that were not scheduled in the previous
# step. Such paths did not have runtime information associated, likely due to
# implementation status.
for index, key in enumerate(paths.keys()):
# Append both the key and value in case the value is empty.
chunked_manifests[index % chunks][0].append(key)
# One last sort by the runtime, then number of test paths.
chunked_manifests.sort(key=lambda x: (x[1], len(x[0])))
# Return just the chunked test paths.
return [c[0] for c in chunked_manifests]
def run_mochitest_general(command_context,
flavor=None,
test_objects=None,
resolve_tests=True,
**kwargs):
from mochitest_options import ALL_FLAVORS
from mozlog.commandline import setup_logging
from mozlog.handlers import StreamHandler
from moztest.resolve import get_suite_definition
# TODO: This is only strictly necessary while mochitest is using Python
# 2 and can be removed once the command is migrated to Python 3.
command_context.activate_virtualenv()
buildapp = None
for app in SUPPORTED_APPS:
if conditions.is_buildapp_in(command_context, apps=[app]):
buildapp = app
break
flavors = None
if flavor:
for fname, fobj in six.iteritems(ALL_FLAVORS):
if flavor in fobj["aliases"]:
if buildapp not in fobj["enabled_apps"]:
continue
flavors = [fname]
break
else:
flavors = [
f for f, v in six.iteritems(ALL_FLAVORS)
if buildapp in v["enabled_apps"]
]
from mozbuild.controller.building import BuildDriver
command_context._ensure_state_subdir_exists(".")
test_paths = kwargs["test_paths"]
kwargs["test_paths"] = []
if kwargs.get("debugger", None):
import mozdebug
if not mozdebug.get_debugger_info(kwargs.get("debugger")):
sys.exit(1)
mochitest = command_context._spawn(MochitestRunner)
tests = []
if resolve_tests:
tests = mochitest.resolve_tests(test_paths,
test_objects,
cwd=command_context._mach_context.cwd)
if not kwargs.get("log"):
# Create shared logger
format_args = {
"level": command_context._mach_context.settings["test"]["level"]
}
if len(tests) == 1:
format_args["verbose"] = True
format_args["compact"] = False
default_format = command_context._mach_context.settings["test"][
"format"]
kwargs["log"] = setup_logging("mach-mochitest", kwargs,
{default_format: sys.stdout},
format_args)
for handler in kwargs["log"].handlers:
if isinstance(handler, StreamHandler):
handler.formatter.inner.summary_on_shutdown = True
driver = command_context._spawn(BuildDriver)
driver.install_tests()
subsuite = kwargs.get("subsuite")
if subsuite == "default":
kwargs["subsuite"] = None
suites = defaultdict(list)
is_webrtc_tag_present = False
unsupported = set()
for test in tests:
# Check if we're running a webrtc test so we can enable webrtc
# specific test logic later if needed.
if "webrtc" in test.get("tags", ""):
is_webrtc_tag_present = True
# Filter out non-mochitests and unsupported flavors.
if test["flavor"] not in ALL_FLAVORS:
continue
key = (test["flavor"], test.get("subsuite", ""))
if test["flavor"] not in flavors:
unsupported.add(key)
continue
if subsuite == "default":
# "--subsuite default" means only run tests that don't have a subsuite
if test.get("subsuite"):
unsupported.add(key)
continue
elif subsuite and test.get("subsuite", "") != subsuite:
unsupported.add(key)
continue
suites[key].append(test)
# Only webrtc mochitests in the media suite need the websocketprocessbridge.
if ("mochitest", "media") in suites and is_webrtc_tag_present:
req = os.path.join(
"testing",
"tools",
"websocketprocessbridge",
"websocketprocessbridge_requirements_3.txt",
)
command_context.virtualenv_manager.activate()
command_context.virtualenv_manager.install_pip_requirements(
req, require_hashes=False)
# sys.executable is used to start the websocketprocessbridge, though for some
# reason it doesn't get set when calling `activate_this.py` in the virtualenv.
sys.executable = command_context.virtualenv_manager.python_path
# This is a hack to introduce an option in mach to not send
# filtered tests to the mochitest harness. Mochitest harness will read
# the master manifest in that case.
if not resolve_tests:
for flavor in flavors:
key = (flavor, kwargs.get("subsuite"))
suites[key] = []
if not suites:
# Make it very clear why no tests were found
if not unsupported:
print(
TESTS_NOT_FOUND.format("\n".join(
sorted(list(test_paths or test_objects)))))
return 1
msg = []
for f, s in unsupported:
fobj = ALL_FLAVORS[f]
apps = fobj["enabled_apps"]
name = fobj["aliases"][0]
if s:
name = "{} --subsuite {}".format(name, s)
if buildapp not in apps:
reason = "requires {}".format(" or ".join(apps))
else:
reason = "excluded by the command line"
msg.append(" mochitest -f {} ({})".format(name, reason))
print(
SUPPORTED_TESTS_NOT_FOUND.format(buildapp, "\n".join(sorted(msg))))
return 1
if buildapp == "android":
from mozrunner.devices.android_device import (
get_adb_path,
verify_android_device,
InstallIntent,
)
app = kwargs.get("app")
if not app:
app = "org.mozilla.geckoview.test"
device_serial = kwargs.get("deviceSerial")
install = InstallIntent.NO if kwargs.get(
"no_install") else InstallIntent.YES
# verify installation
verify_android_device(
command_context,
install=install,
xre=False,
network=True,
app=app,
device_serial=device_serial,
)
if not kwargs["adbPath"]:
kwargs["adbPath"] = get_adb_path(command_context)
run_mochitest = mochitest.run_android_test
else:
run_mochitest = mochitest.run_desktop_test
overall = None
for (flavor, subsuite), tests in sorted(suites.items()):
suite_name, suite = get_suite_definition(flavor, subsuite)
if "test_paths" in suite["kwargs"]:
del suite["kwargs"]["test_paths"]
harness_args = kwargs.copy()
harness_args.update(suite["kwargs"])
# Pass in the full suite name as defined in moztest/resolve.py in case
# chunk-by-runtime is called, in which case runtime information for
# specific mochitest suite has to be loaded. See Bug 1637463.
harness_args.update({"suite_name": suite_name})
result = run_mochitest(command_context._mach_context,
tests=tests,
**harness_args)
if result:
overall = result
# Halt tests on keyboard interrupt
if result == -1:
break
# Only shutdown the logger if we created it
if kwargs["log"].name == "mach-mochitest":
kwargs["log"].shutdown()
return overall