def gen_test_backend():
build_obj = MozbuildObject.from_environment()
try:
config = build_obj.config_environment
except BuildEnvironmentNotFoundException:
# Create a stub config.status file, since the TestManifest backend needs
# to be re-created if configure runs. If the file doesn't exist,
# mozbuild continually thinks the TestManifest backend is out of date
# and tries to regenerate it.
if not os.path.isdir(build_obj.topobjdir):
os.makedirs(build_obj.topobjdir)
config_status = mozpath.join(build_obj.topobjdir, "config.status")
open(config_status, "w").close()
print("No build detected, test metadata may be incomplete.")
# If 'JS_STANDALONE' is set, tests that don't require an objdir won't
# be picked up due to bug 1345209.
substs = EmptyConfig.default_substs
if "JS_STANDALONE" in substs:
del substs["JS_STANDALONE"]
config = EmptyConfig(build_obj.topsrcdir, substs)
config.topobjdir = build_obj.topobjdir
reader = BuildReader(config)
emitter = TreeMetadataEmitter(config)
backend = TestManifestBackend(config)
context = reader.read_topsrcdir()
data = emitter.emit(context, emitfn=emitter._process_test_manifests)
backend.consume(data)
def find_paths_and_tags(self, verbose):
paths, tags = set(), set()
changed_files = self.find_changed_files()
if changed_files:
if verbose:
print("Pushing tests based on modifications to the "
"following files:\n\t%s" % "\n\t".join(changed_files))
from mozbuild.frontend.reader import (
BuildReader,
EmptyConfig,
)
config = EmptyConfig(self.topsrcdir)
reader = BuildReader(config)
files_info = reader.files_info(changed_files)
for path, info in files_info.items():
paths |= info.test_files
tags |= info.test_tags
if verbose:
if paths:
print("Pushing tests based on the following patterns:\n\t%s" %
"\n\t".join(paths))
if tags:
print("Pushing tests based on the following tags:\n\t%s" %
"\n\t".join(tags))
return paths, tags
def find_paths_and_metadata(self, verbose, detect_paths):
paths, tags, flavors = set(), set(), set()
changed_files = self.vcs.files_changed
if changed_files and detect_paths:
if verbose:
print("Pushing tests based on modifications to the "
"following files:\n\t%s" % "\n\t".join(changed_files))
from mozbuild.frontend.reader import (
BuildReader,
EmptyConfig,
)
config = EmptyConfig(self.topsrcdir)
reader = BuildReader(config)
files_info = reader.files_info(changed_files)
for path, info in files_info.items():
paths |= info.test_files
tags |= info.test_tags
flavors |= info.test_flavors
if verbose:
if paths:
print("Pushing tests based on the following patterns:\n\t%s" %
"\n\t".join(paths))
if tags:
print("Pushing tests based on the following tags:\n\t%s" %
"\n\t".join(tags))
return {
'paths': paths,
'tags': tags,
'flavors': flavors,
}
def gen_test_backend():
build_obj = MozbuildObject.from_environment()
try:
config = build_obj.config_environment
except BuildEnvironmentNotFoundException:
print("No build detected, test metadata may be incomplete.")
config = EmptyConfig(build_obj.topsrcdir)
config.topobjdir = build_obj.topobjdir
reader = BuildReader(config)
emitter = TreeMetadataEmitter(config)
backend = TestManifestBackend(config)
context = reader.read_topsrcdir()
data = emitter.emit(context, emitfn=emitter._process_test_manifests)
backend.consume(data)
def _get_reader(self, finder):
from mozbuild.frontend.reader import (
BuildReader,
EmptyConfig,
)
config = EmptyConfig(self.topsrcdir)
return BuildReader(config, finder=finder)
def file_info_schedules(self, paths):
"""Show what is scheduled by the given files.
Given a requested set of files (which can be specified using
wildcards), print the total set of scheduled components.
"""
from mozbuild.frontend.reader import EmptyConfig, BuildReader
config = EmptyConfig(TOPSRCDIR)
reader = BuildReader(config)
schedules = set()
for p, m in reader.files_info(paths).items():
schedules |= set(m['SCHEDULES'].components)
print(", ".join(schedules))
def gen_test_backend():
build_obj = MozbuildObject.from_environment()
try:
config = build_obj.config_environment
except BuildEnvironmentNotFoundException:
print("No build detected, test metadata may be incomplete.")
# If 'JS_STANDALONE' is set, tests that don't require an objdir won't
# be picked up due to bug 1345209.
substs = EmptyConfig.default_substs
if 'JS_STANDALONE' in substs:
del substs['JS_STANDALONE']
config = EmptyConfig(build_obj.topsrcdir, substs)
config.topobjdir = build_obj.topobjdir
reader = BuildReader(config)
emitter = TreeMetadataEmitter(config)
backend = TestManifestBackend(config)
context = reader.read_topsrcdir()
data = emitter.emit(context, emitfn=emitter._process_test_manifests)
backend.consume(data)
def test_filesystem_traversal_no_config(self):
"""Reading moz.build files via filesystem traversal mode with no build config.
This is similar to the above test except no build config is applied.
This will likely fail in more scenarios than the above test because a
lot of moz.build files assumes certain variables are present.
"""
here = os.path.abspath(os.path.dirname(__file__))
root = os.path.normpath(os.path.join(here, '..', '..'))
config = EmptyConfig(root)
reader = BuildReader(config)
all_paths = self._mozbuilds(reader)
paths, contexts = reader.read_relevant_mozbuilds(all_paths)
self.assertEqual(set(paths.keys()), all_paths)
self.assertGreaterEqual(len(contexts), len(paths))
def get_outgoing_metadata(self):
paths, tags, flavors = set(), set(), set()
changed_files = self.vcs.get_outgoing_files('AM')
if changed_files:
config = EmptyConfig(self.topsrcdir)
reader = BuildReader(config)
files_info = reader.files_info(changed_files)
for path, info in files_info.items():
paths |= info.test_files
tags |= info.test_tags
flavors |= info.test_flavors
return {
'paths': paths,
'tags': tags,
'flavors': flavors,
}
def mozbuild_reader(self,
config_mode='build',
vcs_revision=None,
vcs_check_clean=True):
"""Obtain a ``BuildReader`` for evaluating moz.build files.
Given arguments, returns a ``mozbuild.frontend.reader.BuildReader``
that can be used to evaluate moz.build files for this repo.
``config_mode`` is either ``build`` or ``empty``. If ``build``,
``self.config_environment`` is used. This requires a configured build
system to work. If ``empty``, an empty config is used. ``empty`` is
appropriate for file-based traversal mode where ``Files`` metadata is
read.
If ``vcs_revision`` is defined, it specifies a version control revision
to use to obtain files content. The default is to use the filesystem.
This mode is only supported with Mercurial repositories.
If ``vcs_revision`` is not defined and the version control checkout is
sparse, this implies ``vcs_revision='.'``.
If ``vcs_revision`` is ``.`` (denotes the parent of the working
directory), we will verify that the working directory is clean unless
``vcs_check_clean`` is False. This prevents confusion due to uncommitted
file changes not being reflected in the reader.
"""
from mozbuild.frontend.reader import (
default_finder,
BuildReader,
EmptyConfig,
)
from mozpack.files import (
MercurialRevisionFinder, )
if config_mode == 'build':
config = self.config_environment
elif config_mode == 'empty':
config = EmptyConfig(self.topsrcdir)
else:
raise ValueError('unknown config_mode value: %s' % config_mode)
try:
repo = self.repository
except InvalidRepoPath:
repo = None
if repo and not vcs_revision and repo.sparse_checkout_present():
vcs_revision = '.'
if vcs_revision is None:
finder = default_finder
else:
# If we failed to detect the repo prior, check again to raise its
# exception.
if not repo:
self.repository
assert False
if repo.name != 'hg':
raise Exception('do not support VCS reading mode for %s' %
repo.name)
if vcs_revision == '.' and vcs_check_clean:
with repo:
if not repo.working_directory_clean():
raise Exception('working directory is not clean; '
'refusing to use a VCS-based finder')
finder = MercurialRevisionFinder(self.topsrcdir,
rev=vcs_revision,
recognize_repo_paths=True)
return BuildReader(config, finder=finder)
def test(self, what):
"""Run tests from names or paths.
mach test accepts arguments specifying which tests to run. Each argument
can be:
* The path to a test file
* A directory containing tests
* A test suite name
* An alias to a test suite name (codes used on TreeHerder)
If no input is provided, tests will be run based on files changed in
the local tree. Relevant tests, tags, or flavors are determined by
IMPACTED_TESTS annotations in moz.build files relevant to the
changed files.
When paths or directories are given, they are first resolved to test
files known to the build system.
If resolved tests belong to more than one test type/flavor/harness,
the harness for each relevant type/flavor will be invoked. e.g. if
you specify a directory with xpcshell and browser chrome mochitests,
both harnesses will be invoked.
"""
from mozbuild.testing import TestResolver
# Parse arguments and assemble a test "plan."
run_suites = set()
run_tests = []
resolver = self._spawn(TestResolver)
for entry in what:
# If the path matches the name or alias of an entire suite, run
# the entire suite.
if entry in TEST_SUITES:
run_suites.add(entry)
continue
suitefound = False
for suite, v in TEST_SUITES.items():
if entry in v.get('aliases', []):
run_suites.add(suite)
suitefound = True
if suitefound:
continue
# Now look for file/directory matches in the TestResolver.
relpath = self._wrap_path_argument(entry).relpath()
tests = list(resolver.resolve_tests(paths=[relpath]))
run_tests.extend(tests)
if not tests:
print('UNKNOWN TEST: %s' % entry, file=sys.stderr)
if not what:
# TODO: This isn't really related to try, and should be
# extracted to a common library for vcs interactions when it is
# introduced in bug 1185599.
from autotry import AutoTry
at = AutoTry(self.topsrcdir, resolver, self._mach_context)
changed_files = at.find_changed_files()
if changed_files:
print("Tests will be run based on modifications to the "
"following files:\n\t%s" % "\n\t".join(changed_files))
from mozbuild.frontend.reader import (
BuildReader,
EmptyConfig,
)
config = EmptyConfig(self.topsrcdir)
reader = BuildReader(config)
files_info = reader.files_info(changed_files)
paths, tags, flavors = set(), set(), set()
for info in files_info.values():
paths |= info.test_files
tags |= info.test_tags
flavors |= info.test_flavors
# This requires multiple calls to resolve_tests, because the test
# resolver returns tests that match every condition, while we want
# tests that match any condition. Bug 1210213 tracks implementing
# more flexible querying.
if tags:
run_tests = list(resolver.resolve_tests(tags=tags))
if paths:
run_tests += [
t for t in resolver.resolve_tests(paths=paths)
if not (tags & set(t.get('tags', '').split()))
]
if flavors:
run_tests = [
t for t in run_tests if t['flavor'] not in flavors
]
for flavor in flavors:
run_tests += list(resolver.resolve_tests(flavor=flavor))
if not run_suites and not run_tests:
print(UNKNOWN_TEST)
return 1
status = None
for suite_name in run_suites:
suite = TEST_SUITES[suite_name]
if 'mach_command' in suite:
res = self._mach_context.commands.dispatch(
suite['mach_command'], self._mach_context,
**suite['kwargs'])
if res:
status = res
buckets = {}
for test in run_tests:
key = (test['flavor'], test.get('subsuite', ''))
buckets.setdefault(key, []).append(test)
for (flavor, subsuite), tests in sorted(buckets.items()):
if flavor not in TEST_FLAVORS:
print(UNKNOWN_FLAVOR % flavor)
status = 1
continue
m = TEST_FLAVORS[flavor]
if 'mach_command' not in m:
print(UNKNOWN_FLAVOR % flavor)
status = 1
continue
kwargs = dict(m['kwargs'])
kwargs['subsuite'] = subsuite
res = self._mach_context.commands.dispatch(m['mach_command'],
self._mach_context,
test_objects=tests,
**kwargs)
if res:
status = res
return status
def _get_reader(self):
from mozbuild.frontend.reader import BuildReader, EmptyConfig
config = EmptyConfig(self.topsrcdir)
return BuildReader(config)