def build_docs(self, format=None, outdir=None):
self._activate_virtualenv()
self.virtualenv_manager.install_pip_package('mdn-sphinx-theme==0.4')
from moztreedocs import SphinxManager
if outdir == '<DEFAULT>':
outdir = os.path.join(self.topobjdir, 'docs')
manager = SphinxManager(self.topsrcdir,
os.path.join(self.topsrcdir, 'tools', 'docs'),
outdir)
# We don't care about GYP projects, so don't process them. This makes
# scanning faster and may even prevent an exception.
def remove_gyp_dirs(sandbox):
sandbox['GYP_DIRS'][:] = []
reader = BuildReader(self.config_environment,
sandbox_post_eval_cb=remove_gyp_dirs)
for sandbox in reader.walk_topsrcdir():
for dest_dir, source_dir in sandbox['SPHINX_TREES'].items():
manager.add_tree(
os.path.join(sandbox['RELATIVEDIR'], source_dir), dest_dir)
for entry in sandbox['SPHINX_PYTHON_PACKAGE_DIRS']:
manager.add_python_package_dir(
os.path.join(sandbox['RELATIVEDIR'], entry))
return manager.generate_docs(format)
def test_orphan_file_patterns(self):
if sys.platform == 'win32':
raise unittest.SkipTest('failing on windows builds')
mb = MozbuildObject.from_environment(detect_virtualenv_mozinfo=False)
try:
config = mb.config_environment
except Exception as e:
if str(e) == 'config.status not available. Run configure.':
raise unittest.SkipTest('failing without config.status')
raise
if config.substs['MOZ_BUILD_APP'] == 'js':
raise unittest.SkipTest('failing in Spidermonkey builds')
reader = BuildReader(config)
all_paths = self._mozbuilds(reader)
_, contexts = reader.read_relevant_mozbuilds(all_paths)
finder = FileFinder(config.topsrcdir, ignore=['obj*'])
def pattern_exists(pat):
return [p for p in finder.find(pat)] != []
for ctx in contexts:
if not isinstance(ctx, Files):
continue
relsrcdir = ctx.relsrcdir
for p in ctx.patterns:
if not pattern_exists(os.path.join(relsrcdir, p)):
self.fail("The pattern '%s' in a Files() entry in "
"'%s' corresponds to no files in the tree.\n"
"Please update this entry." %
(p, ctx.main_path))
def build_docs(self, format=None, outdir=None):
self._activate_virtualenv()
self.virtualenv_manager.install_pip_package('mdn-sphinx-theme==0.4')
from moztreedocs import SphinxManager
if outdir == '<DEFAULT>':
outdir = os.path.join(self.topobjdir, 'docs')
manager = SphinxManager(self.topsrcdir,
os.path.join(self.topsrcdir, 'tools', 'docs'),
outdir)
# We don't care about GYP projects, so don't process them. This makes
# scanning faster and may even prevent an exception.
def remove_gyp_dirs(context):
context['GYP_DIRS'][:] = []
reader = BuildReader(self.config_environment,
sandbox_post_eval_cb=remove_gyp_dirs)
for path, name, key, value in reader.find_sphinx_variables():
reldir = os.path.dirname(path)
if name == 'SPHINX_TREES':
assert key
manager.add_tree(os.path.join(reldir, value),
os.path.join(reldir, key))
if name == 'SPHINX_PYTHON_PACKAGE_DIRS':
manager.add_python_package_dir(os.path.join(reldir, value))
return manager.generate_docs(format)
def read_build_config(self):
"""Read the active build config and add docs to this instance."""
# Reading the Sphinx variables doesn't require a full build context.
# Only define the parts we need.
class fakeconfig(object):
def __init__(self, topsrcdir):
self.topsrcdir = topsrcdir
config = fakeconfig(self._topsrcdir)
reader = BuildReader(config)
for path, name, key, value in reader.find_sphinx_variables():
reldir = os.path.dirname(path)
if name == 'SPHINX_TREES':
assert key
if key.startswith('/'):
key = key[1:]
else:
key = os.path.join(reldir, key)
self.add_tree(os.path.join(reldir, value), key)
if name == 'SPHINX_PYTHON_PACKAGE_DIRS':
self.add_python_package_dir(os.path.join(reldir, value))
def gen_test_backend():
build_obj = MozbuildObject.from_environment()
try:
config = build_obj.config_environment
except BuildEnvironmentNotFoundException:
# Create a stub config.status file, since the TestManifest backend needs
# to be re-created if configure runs. If the file doesn't exist,
# mozbuild continually thinks the TestManifest backend is out of date
# and tries to regenerate it.
if not os.path.isdir(build_obj.topobjdir):
os.makedirs(build_obj.topobjdir)
config_status = mozpath.join(build_obj.topobjdir, "config.status")
open(config_status, "w").close()
print("No build detected, test metadata may be incomplete.")
# If 'JS_STANDALONE' is set, tests that don't require an objdir won't
# be picked up due to bug 1345209.
substs = EmptyConfig.default_substs
if "JS_STANDALONE" in substs:
del substs["JS_STANDALONE"]
config = EmptyConfig(build_obj.topsrcdir, substs)
config.topobjdir = build_obj.topobjdir
reader = BuildReader(config)
emitter = TreeMetadataEmitter(config)
backend = TestManifestBackend(config)
context = reader.read_topsrcdir()
data = emitter.emit(context, emitfn=emitter._process_test_manifests)
backend.consume(data)
def resolver(request, tmpdir, monkeypatch, topsrcdir, all_tests, defaults):
topobjdir = tmpdir.mkdir("objdir").strpath
loader_cls = request.param
if loader_cls == BuildBackendLoader:
with open(os.path.join(topobjdir, "all-tests.pkl"), "wb") as fh:
pickle.dump(all_tests, fh)
with open(os.path.join(topobjdir, "test-defaults.pkl"), "wb") as fh:
pickle.dump(defaults, fh)
# The mock data already exists, so prevent BuildBackendLoader from regenerating
# the build information from the whole gecko tree...
class BuildBackendLoaderNeverOutOfDate(BuildBackendLoader):
def backend_out_of_date(self, backend_file):
return False
loader_cls = BuildBackendLoaderNeverOutOfDate
# Patch WPT's manifestupdate.run to return tests based on the contents of
# 'data/srcdir/wpt_manifest_data.json'.
monkeypatch.setattr(manifestupdate, "run", fake_wpt_manifestupdate)
resolver = TestResolver(topsrcdir,
None,
None,
topobjdir=topobjdir,
loader_cls=loader_cls)
resolver._puppeteer_loaded = True
if loader_cls == TestManifestLoader:
config = MockConfig(topsrcdir)
resolver.load_tests.reader = BuildReader(config)
return resolver
def reader(self, name, enable_tests=False):
config = self.config(name)
if enable_tests:
config.substs['ENABLE_TESTS'] = '1'
return BuildReader(config)
def resolver(request, tmpdir, topsrcdir, all_tests, defaults):
topobjdir = tmpdir.mkdir("objdir").strpath
loader_cls = request.param
if loader_cls == BuildBackendLoader:
with open(os.path.join(topobjdir, 'all-tests.pkl'), 'wb') as fh:
pickle.dump(all_tests, fh)
with open(os.path.join(topobjdir, 'test-defaults.pkl'), 'wb') as fh:
pickle.dump(defaults, fh)
# The mock data already exists, so prevent BuildBackendLoader from regenerating
# the build information from the whole gecko tree...
class BuildBackendLoaderNeverOutOfDate(BuildBackendLoader):
def backend_out_of_date(self, backend_file):
return False
loader_cls = BuildBackendLoaderNeverOutOfDate
resolver = TestResolver(topsrcdir,
None,
None,
topobjdir=topobjdir,
loader_cls=loader_cls)
resolver._puppeteer_loaded = True
resolver._wpt_loaded = True
if loader_cls == TestManifestLoader:
config = MockConfig(topsrcdir)
resolver.load_tests.reader = BuildReader(config)
return resolver
def reader(self, name):
config = MockConfig(os.path.join(data_path, name), extra_substs=dict(
ENABLE_TESTS='1',
BIN_SUFFIX='.prog',
))
return BuildReader(config)
def reader(self, name, enable_tests=False, **kwargs):
extra = {}
if enable_tests:
extra['ENABLE_TESTS'] = '1'
config = self.config(name, extra_substs=extra)
return BuildReader(config, **kwargs)
def find_paths_and_metadata(self, verbose, detect_paths):
paths, tags, flavors = set(), set(), set()
changed_files = self.vcs.files_changed
if changed_files and detect_paths:
if verbose:
print("Pushing tests based on modifications to the "
"following files:\n\t%s" % "\n\t".join(changed_files))
from mozbuild.frontend.reader import (
BuildReader,
EmptyConfig,
)
config = EmptyConfig(self.topsrcdir)
reader = BuildReader(config)
files_info = reader.files_info(changed_files)
for path, info in files_info.items():
paths |= info.test_files
tags |= info.test_tags
flavors |= info.test_flavors
if verbose:
if paths:
print("Pushing tests based on the following patterns:\n\t%s" %
"\n\t".join(paths))
if tags:
print("Pushing tests based on the following tags:\n\t%s" %
"\n\t".join(tags))
return {
'paths': paths,
'tags': tags,
'flavors': flavors,
}
def find_paths_and_tags(self, verbose):
paths, tags = set(), set()
changed_files = self.find_changed_files()
if changed_files:
if verbose:
print("Pushing tests based on modifications to the "
"following files:\n\t%s" % "\n\t".join(changed_files))
from mozbuild.frontend.reader import (
BuildReader,
EmptyConfig,
)
config = EmptyConfig(self.topsrcdir)
reader = BuildReader(config)
files_info = reader.files_info(changed_files)
for path, info in files_info.items():
paths |= info.test_files
tags |= info.test_tags
if verbose:
if paths:
print("Pushing tests based on the following patterns:\n\t%s" %
"\n\t".join(paths))
if tags:
print("Pushing tests based on the following tags:\n\t%s" %
"\n\t".join(tags))
return paths, tags
def _get_reader(self, finder):
from mozbuild.frontend.reader import (
BuildReader,
EmptyConfig,
)
config = EmptyConfig(self.topsrcdir)
return BuildReader(config, finder=finder)
def reader(self, name, enable_tests=False, error_is_fatal=True, **kwargs):
extra = {}
if enable_tests:
extra["ENABLE_TESTS"] = "1"
config = self.config(name,
extra_substs=extra,
error_is_fatal=error_is_fatal)
return BuildReader(config, **kwargs)
def reader(self, name, enable_tests=False):
config = MockConfig(mozpath.join(data_path, name),
extra_substs=dict(
ENABLE_TESTS='1' if enable_tests else '',
BIN_SUFFIX='.prog',
OS_TARGET='WINNT',
))
return BuildReader(config)
def reader(self, name, enable_tests=False, extra_substs=None):
substs = dict(
ENABLE_TESTS='1' if enable_tests else '',
BIN_SUFFIX='.prog',
OS_TARGET='WINNT',
COMPILE_ENVIRONMENT='1',
)
if extra_substs:
substs.update(extra_substs)
config = MockConfig(mozpath.join(data_path, name), extra_substs=substs)
return BuildReader(config)
def test_filesystem_traversal_reading(self):
"""Reading moz.build according to filesystem traversal works.
We attempt to read every known moz.build file via filesystem traversal.
If this test fails, it means that metadata extraction will fail.
"""
mb = MozbuildObject.from_environment(detect_virtualenv_mozinfo=False)
config = mb.config_environment
reader = BuildReader(config)
all_paths = self._mozbuilds(reader)
paths, contexts = reader.read_relevant_mozbuilds(all_paths)
self.assertEqual(set(paths), all_paths)
self.assertGreaterEqual(len(contexts), len(paths))
def file_info_schedules(self, paths):
"""Show what is scheduled by the given files.
Given a requested set of files (which can be specified using
wildcards), print the total set of scheduled components.
"""
from mozbuild.frontend.reader import EmptyConfig, BuildReader
config = EmptyConfig(TOPSRCDIR)
reader = BuildReader(config)
schedules = set()
for p, m in reader.files_info(paths).items():
schedules |= set(m['SCHEDULES'].components)
print(", ".join(schedules))
def test_filesystem_traversal_no_config(self):
"""Reading moz.build files via filesystem traversal mode with no build config.
This is similar to the above test except no build config is applied.
This will likely fail in more scenarios than the above test because a
lot of moz.build files assumes certain variables are present.
"""
here = os.path.abspath(os.path.dirname(__file__))
root = os.path.normpath(os.path.join(here, '..', '..'))
config = EmptyConfig(root)
reader = BuildReader(config)
all_paths = self._mozbuilds(reader)
paths, contexts = reader.read_relevant_mozbuilds(all_paths)
self.assertEqual(set(paths.keys()), all_paths)
self.assertGreaterEqual(len(contexts), len(paths))
def gen_test_backend():
build_obj = MozbuildObject.from_environment()
try:
config = build_obj.config_environment
except BuildEnvironmentNotFoundException:
print("No build detected, test metadata may be incomplete.")
config = EmptyConfig(build_obj.topsrcdir)
config.topobjdir = build_obj.topobjdir
reader = BuildReader(config)
emitter = TreeMetadataEmitter(config)
backend = TestManifestBackend(config)
context = reader.read_topsrcdir()
data = emitter.emit(context, emitfn=emitter._process_test_manifests)
backend.consume(data)
def test_mtime_no_change(self):
"""Ensure mtime is not updated if file content does not change."""
env = self._consume('stub0', RecursiveMakeBackend)
makefile_path = mozpath.join(env.topobjdir, 'Makefile')
backend_path = mozpath.join(env.topobjdir, 'backend.mk')
makefile_mtime = os.path.getmtime(makefile_path)
backend_mtime = os.path.getmtime(backend_path)
reader = BuildReader(env)
emitter = TreeMetadataEmitter(env)
backend = RecursiveMakeBackend(env)
backend.consume(emitter.emit(reader.read_topsrcdir()))
self.assertEqual(os.path.getmtime(makefile_path), makefile_mtime)
self.assertEqual(os.path.getmtime(backend_path), backend_mtime)
def read_build_config(docdir):
"""Read the active build config and return the relevant doc paths.
The return value is cached so re-generating with the same docdir won't
invoke the build system a second time."""
trees = {}
python_package_dirs = set()
is_main = docdir == MAIN_DOC_PATH
relevant_mozbuild_path = None if is_main else docdir
# Reading the Sphinx variables doesn't require a full build context.
# Only define the parts we need.
class fakeconfig(object):
topsrcdir = build.topsrcdir
variables = ("SPHINX_TREES", "SPHINX_PYTHON_PACKAGE_DIRS")
reader = BuildReader(fakeconfig())
result = reader.find_variables_from_ast(variables,
path=relevant_mozbuild_path)
for path, name, key, value in result:
reldir = os.path.dirname(path)
if name == "SPHINX_TREES":
# If we're building a subtree, only process that specific subtree.
# topsrcdir always uses POSIX-style path, normalize it for proper comparison.
absdir = os.path.normpath(
os.path.join(build.topsrcdir, reldir, value))
if not is_main and absdir not in (docdir, MAIN_DOC_PATH):
continue
assert key
if key.startswith("/"):
key = key[1:]
else:
key = os.path.normpath(os.path.join(reldir, key))
if key in trees:
raise Exception(
"%s has already been registered as a destination." % key)
trees[key] = os.path.join(reldir, value)
if name == "SPHINX_PYTHON_PACKAGE_DIRS":
python_package_dirs.add(os.path.join(reldir, value))
return trees, python_package_dirs
def resolver(request, tmpdir, topsrcdir, all_tests, defaults):
topobjdir = tmpdir.mkdir("objdir").strpath
loader_cls = request.param
if loader_cls == BuildBackendLoader:
with open(os.path.join(topobjdir, 'all-tests.pkl'), 'wb') as fh:
pickle.dump(all_tests, fh)
with open(os.path.join(topobjdir, 'test-defaults.pkl'), 'wb') as fh:
pickle.dump(defaults, fh)
resolver = TestResolver(topsrcdir, None, None, topobjdir=topobjdir, loader_cls=loader_cls)
resolver._puppeteer_loaded = True
resolver._wpt_loaded = True
if loader_cls == TestManifestLoader:
config = MockConfig(topsrcdir)
resolver.load_tests.reader = BuildReader(config)
return resolver
def get_outgoing_metadata(self):
paths, tags, flavors = set(), set(), set()
changed_files = self.vcs.get_outgoing_files('AM')
if changed_files:
config = EmptyConfig(self.topsrcdir)
reader = BuildReader(config)
files_info = reader.files_info(changed_files)
for path, info in files_info.items():
paths |= info.test_files
tags |= info.test_tags
flavors |= info.test_flavors
return {
'paths': paths,
'tags': tags,
'flavors': flavors,
}
def do_test_backend(self, *backends, **kwargs):
topobjdir = mkdtemp()
try:
config = ConfigEnvironment(buildconfig.topsrcdir, topobjdir,
**kwargs)
reader = BuildReader(config)
emitter = TreeMetadataEmitter(config)
moz_build = mozpath.join(config.topsrcdir, 'test.mozbuild')
definitions = list(emitter.emit(
reader.read_mozbuild(moz_build, config)))
for backend in backends:
backend(config).consume(definitions)
yield config
except:
raise
finally:
if not os.environ.get('MOZ_NO_CLEANUP'):
shutil.rmtree(topobjdir)
def do_test_backend(self, *backends, **kwargs):
# Create the objdir in the srcdir to ensure that they share
# the same drive on Windows.
topobjdir = mkdtemp(dir=buildconfig.topsrcdir)
try:
config = ConfigEnvironment(buildconfig.topsrcdir, topobjdir,
**kwargs)
reader = BuildReader(config)
emitter = TreeMetadataEmitter(config)
moz_build = mozpath.join(config.topsrcdir, "test.mozbuild")
definitions = list(
emitter.emit(reader.read_mozbuild(moz_build, config)))
for backend in backends:
backend(config).consume(definitions)
yield config
except Exception:
raise
finally:
if not os.environ.get("MOZ_NO_CLEANUP"):
shutil.rmtree(topobjdir)
def read_build_config(docdir):
"""Read the active build config and return the relevant doc paths.
The return value is cached so re-generating with the same docdir won't
invoke the build system a second time."""
trees = {}
python_package_dirs = set()
is_main = docdir == MAIN_DOC_PATH
relevant_mozbuild_path = None if is_main else docdir
# Reading the Sphinx variables doesn't require a full build context.
# Only define the parts we need.
class fakeconfig(object):
topsrcdir = build.topsrcdir
reader = BuildReader(fakeconfig())
for path, name, key, value in reader.find_sphinx_variables(relevant_mozbuild_path):
reldir = os.path.join(os.path.dirname(path), value)
if name == 'SPHINX_TREES':
# If we're building a subtree, only process that specific subtree.
absdir = os.path.join(build.topsrcdir, reldir)
if not is_main and absdir not in (docdir, MAIN_DOC_PATH):
continue
assert key
if key.startswith('/'):
key = key[1:]
else:
key = os.path.join(reldir, key)
if key in trees:
raise Exception('%s has already been registered as a destination.' % key)
trees[key] = reldir
if name == 'SPHINX_PYTHON_PACKAGE_DIRS':
python_package_dirs.add(reldir)
return trees, python_package_dirs
def test_orphan_file_patterns(self):
if sys.platform == 'win32':
raise unittest.SkipTest('failing on windows builds')
mb = MozbuildObject.from_environment(detect_virtualenv_mozinfo=False)
try:
config = mb.config_environment
except Exception as e:
if e.message == 'config.status not available. Run configure.':
raise unittest.SkipTest('failing without config.status')
raise
reader = BuildReader(config)
all_paths = self._mozbuilds(reader)
_, contexts = reader.read_relevant_mozbuilds(all_paths)
finder = FileFinder(config.topsrcdir,
find_executables=False,
ignore=['obj*'])
def pattern_exists(pat):
return [p for p in finder.find(pat)] != []
for ctx in contexts:
if not isinstance(ctx, Files):
continue
relsrcdir = ctx.relsrcdir
if not pattern_exists(os.path.join(relsrcdir, ctx.pattern)):
self.fail("The pattern '%s' in a Files() entry in "
"'%s' corresponds to no files in the tree.\n"
"Please update this entry." %
(ctx.pattern, ctx.main_path))
test_files = ctx['IMPACTED_TESTS'].files
for p in test_files:
if not pattern_exists(
os.path.relpath(p.full_path, config.topsrcdir)):
self.fail("The pattern '%s' in a dependent tests entry "
"in '%s' corresponds to no files in the tree.\n"
"Please update this entry." % (p, ctx.main_path))
def build_docs(self, format=None, outdir=None):
self._activate_virtualenv()
self.virtualenv_manager.install_pip_package('sphinx_rtd_theme==0.1.6')
from moztreedocs import SphinxManager
if outdir == '<DEFAULT>':
outdir = os.path.join(self.topobjdir, 'docs')
manager = SphinxManager(self.topsrcdir,
os.path.join(self.topsrcdir, 'tools', 'docs'),
outdir)
# We don't care about GYP projects, so don't process them. This makes
# scanning faster and may even prevent an exception.
def remove_gyp_dirs(context):
context['GYP_DIRS'][:] = []
# Reading the Sphinx variables doesn't require a full build context.
# Only define the parts we need.
class fakeconfig(object):
def __init__(self, topsrcdir):
self.topsrcdir = topsrcdir
config = fakeconfig(self.topsrcdir)
reader = BuildReader(config)
for path, name, key, value in reader.find_sphinx_variables():
reldir = os.path.dirname(path)
if name == 'SPHINX_TREES':
assert key
manager.add_tree(os.path.join(reldir, value),
os.path.join(reldir, key))
if name == 'SPHINX_PYTHON_PACKAGE_DIRS':
manager.add_python_package_dir(os.path.join(reldir, value))
return manager.generate_docs(format)
def gen_test_backend():
build_obj = MozbuildObject.from_environment()
try:
config = build_obj.config_environment
except BuildEnvironmentNotFoundException:
print("No build detected, test metadata may be incomplete.")
# If 'JS_STANDALONE' is set, tests that don't require an objdir won't
# be picked up due to bug 1345209.
substs = EmptyConfig.default_substs
if 'JS_STANDALONE' in substs:
del substs['JS_STANDALONE']
config = EmptyConfig(build_obj.topsrcdir, substs)
config.topobjdir = build_obj.topobjdir
reader = BuildReader(config)
emitter = TreeMetadataEmitter(config)
backend = TestManifestBackend(config)
context = reader.read_topsrcdir()
data = emitter.emit(context, emitfn=emitter._process_test_manifests)
backend.consume(data)
