def build_docs(self, format=None, outdir=None): self._activate_virtualenv() self.virtualenv_manager.install_pip_package('mdn-sphinx-theme==0.4') from moztreedocs import SphinxManager if outdir == '<DEFAULT>': outdir = os.path.join(self.topobjdir, 'docs') manager = SphinxManager(self.topsrcdir, os.path.join(self.topsrcdir, 'tools', 'docs'), outdir) # We don't care about GYP projects, so don't process them. This makes # scanning faster and may even prevent an exception. def remove_gyp_dirs(sandbox): sandbox['GYP_DIRS'][:] = [] reader = BuildReader(self.config_environment, sandbox_post_eval_cb=remove_gyp_dirs) for sandbox in reader.walk_topsrcdir(): for dest_dir, source_dir in sandbox['SPHINX_TREES'].items(): manager.add_tree( os.path.join(sandbox['RELATIVEDIR'], source_dir), dest_dir) for entry in sandbox['SPHINX_PYTHON_PACKAGE_DIRS']: manager.add_python_package_dir( os.path.join(sandbox['RELATIVEDIR'], entry)) return manager.generate_docs(format)
def test_orphan_file_patterns(self): if sys.platform == 'win32': raise unittest.SkipTest('failing on windows builds') mb = MozbuildObject.from_environment(detect_virtualenv_mozinfo=False) try: config = mb.config_environment except Exception as e: if str(e) == 'config.status not available. Run configure.': raise unittest.SkipTest('failing without config.status') raise if config.substs['MOZ_BUILD_APP'] == 'js': raise unittest.SkipTest('failing in Spidermonkey builds') reader = BuildReader(config) all_paths = self._mozbuilds(reader) _, contexts = reader.read_relevant_mozbuilds(all_paths) finder = FileFinder(config.topsrcdir, ignore=['obj*']) def pattern_exists(pat): return [p for p in finder.find(pat)] != [] for ctx in contexts: if not isinstance(ctx, Files): continue relsrcdir = ctx.relsrcdir for p in ctx.patterns: if not pattern_exists(os.path.join(relsrcdir, p)): self.fail("The pattern '%s' in a Files() entry in " "'%s' corresponds to no files in the tree.\n" "Please update this entry." % (p, ctx.main_path))
def build_docs(self, format=None, outdir=None): self._activate_virtualenv() self.virtualenv_manager.install_pip_package('mdn-sphinx-theme==0.4') from moztreedocs import SphinxManager if outdir == '<DEFAULT>': outdir = os.path.join(self.topobjdir, 'docs') manager = SphinxManager(self.topsrcdir, os.path.join(self.topsrcdir, 'tools', 'docs'), outdir) # We don't care about GYP projects, so don't process them. This makes # scanning faster and may even prevent an exception. def remove_gyp_dirs(context): context['GYP_DIRS'][:] = [] reader = BuildReader(self.config_environment, sandbox_post_eval_cb=remove_gyp_dirs) for path, name, key, value in reader.find_sphinx_variables(): reldir = os.path.dirname(path) if name == 'SPHINX_TREES': assert key manager.add_tree(os.path.join(reldir, value), os.path.join(reldir, key)) if name == 'SPHINX_PYTHON_PACKAGE_DIRS': manager.add_python_package_dir(os.path.join(reldir, value)) return manager.generate_docs(format)
def read_build_config(self): """Read the active build config and add docs to this instance.""" # Reading the Sphinx variables doesn't require a full build context. # Only define the parts we need. class fakeconfig(object): def __init__(self, topsrcdir): self.topsrcdir = topsrcdir config = fakeconfig(self._topsrcdir) reader = BuildReader(config) for path, name, key, value in reader.find_sphinx_variables(): reldir = os.path.dirname(path) if name == 'SPHINX_TREES': assert key if key.startswith('/'): key = key[1:] else: key = os.path.join(reldir, key) self.add_tree(os.path.join(reldir, value), key) if name == 'SPHINX_PYTHON_PACKAGE_DIRS': self.add_python_package_dir(os.path.join(reldir, value))
def gen_test_backend(): build_obj = MozbuildObject.from_environment() try: config = build_obj.config_environment except BuildEnvironmentNotFoundException: # Create a stub config.status file, since the TestManifest backend needs # to be re-created if configure runs. If the file doesn't exist, # mozbuild continually thinks the TestManifest backend is out of date # and tries to regenerate it. if not os.path.isdir(build_obj.topobjdir): os.makedirs(build_obj.topobjdir) config_status = mozpath.join(build_obj.topobjdir, "config.status") open(config_status, "w").close() print("No build detected, test metadata may be incomplete.") # If 'JS_STANDALONE' is set, tests that don't require an objdir won't # be picked up due to bug 1345209. substs = EmptyConfig.default_substs if "JS_STANDALONE" in substs: del substs["JS_STANDALONE"] config = EmptyConfig(build_obj.topsrcdir, substs) config.topobjdir = build_obj.topobjdir reader = BuildReader(config) emitter = TreeMetadataEmitter(config) backend = TestManifestBackend(config) context = reader.read_topsrcdir() data = emitter.emit(context, emitfn=emitter._process_test_manifests) backend.consume(data)
def resolver(request, tmpdir, monkeypatch, topsrcdir, all_tests, defaults): topobjdir = tmpdir.mkdir("objdir").strpath loader_cls = request.param if loader_cls == BuildBackendLoader: with open(os.path.join(topobjdir, "all-tests.pkl"), "wb") as fh: pickle.dump(all_tests, fh) with open(os.path.join(topobjdir, "test-defaults.pkl"), "wb") as fh: pickle.dump(defaults, fh) # The mock data already exists, so prevent BuildBackendLoader from regenerating # the build information from the whole gecko tree... class BuildBackendLoaderNeverOutOfDate(BuildBackendLoader): def backend_out_of_date(self, backend_file): return False loader_cls = BuildBackendLoaderNeverOutOfDate # Patch WPT's manifestupdate.run to return tests based on the contents of # 'data/srcdir/wpt_manifest_data.json'. monkeypatch.setattr(manifestupdate, "run", fake_wpt_manifestupdate) resolver = TestResolver(topsrcdir, None, None, topobjdir=topobjdir, loader_cls=loader_cls) resolver._puppeteer_loaded = True if loader_cls == TestManifestLoader: config = MockConfig(topsrcdir) resolver.load_tests.reader = BuildReader(config) return resolver
def reader(self, name, enable_tests=False): config = self.config(name) if enable_tests: config.substs['ENABLE_TESTS'] = '1' return BuildReader(config)
def resolver(request, tmpdir, topsrcdir, all_tests, defaults): topobjdir = tmpdir.mkdir("objdir").strpath loader_cls = request.param if loader_cls == BuildBackendLoader: with open(os.path.join(topobjdir, 'all-tests.pkl'), 'wb') as fh: pickle.dump(all_tests, fh) with open(os.path.join(topobjdir, 'test-defaults.pkl'), 'wb') as fh: pickle.dump(defaults, fh) # The mock data already exists, so prevent BuildBackendLoader from regenerating # the build information from the whole gecko tree... class BuildBackendLoaderNeverOutOfDate(BuildBackendLoader): def backend_out_of_date(self, backend_file): return False loader_cls = BuildBackendLoaderNeverOutOfDate resolver = TestResolver(topsrcdir, None, None, topobjdir=topobjdir, loader_cls=loader_cls) resolver._puppeteer_loaded = True resolver._wpt_loaded = True if loader_cls == TestManifestLoader: config = MockConfig(topsrcdir) resolver.load_tests.reader = BuildReader(config) return resolver
def reader(self, name): config = MockConfig(os.path.join(data_path, name), extra_substs=dict( ENABLE_TESTS='1', BIN_SUFFIX='.prog', )) return BuildReader(config)
def reader(self, name, enable_tests=False, **kwargs): extra = {} if enable_tests: extra['ENABLE_TESTS'] = '1' config = self.config(name, extra_substs=extra) return BuildReader(config, **kwargs)
def find_paths_and_metadata(self, verbose, detect_paths): paths, tags, flavors = set(), set(), set() changed_files = self.vcs.files_changed if changed_files and detect_paths: if verbose: print("Pushing tests based on modifications to the " "following files:\n\t%s" % "\n\t".join(changed_files)) from mozbuild.frontend.reader import ( BuildReader, EmptyConfig, ) config = EmptyConfig(self.topsrcdir) reader = BuildReader(config) files_info = reader.files_info(changed_files) for path, info in files_info.items(): paths |= info.test_files tags |= info.test_tags flavors |= info.test_flavors if verbose: if paths: print("Pushing tests based on the following patterns:\n\t%s" % "\n\t".join(paths)) if tags: print("Pushing tests based on the following tags:\n\t%s" % "\n\t".join(tags)) return { 'paths': paths, 'tags': tags, 'flavors': flavors, }
def find_paths_and_tags(self, verbose): paths, tags = set(), set() changed_files = self.find_changed_files() if changed_files: if verbose: print("Pushing tests based on modifications to the " "following files:\n\t%s" % "\n\t".join(changed_files)) from mozbuild.frontend.reader import ( BuildReader, EmptyConfig, ) config = EmptyConfig(self.topsrcdir) reader = BuildReader(config) files_info = reader.files_info(changed_files) for path, info in files_info.items(): paths |= info.test_files tags |= info.test_tags if verbose: if paths: print("Pushing tests based on the following patterns:\n\t%s" % "\n\t".join(paths)) if tags: print("Pushing tests based on the following tags:\n\t%s" % "\n\t".join(tags)) return paths, tags
def _get_reader(self, finder): from mozbuild.frontend.reader import ( BuildReader, EmptyConfig, ) config = EmptyConfig(self.topsrcdir) return BuildReader(config, finder=finder)
def reader(self, name, enable_tests=False, error_is_fatal=True, **kwargs): extra = {} if enable_tests: extra["ENABLE_TESTS"] = "1" config = self.config(name, extra_substs=extra, error_is_fatal=error_is_fatal) return BuildReader(config, **kwargs)
def reader(self, name, enable_tests=False): config = MockConfig(mozpath.join(data_path, name), extra_substs=dict( ENABLE_TESTS='1' if enable_tests else '', BIN_SUFFIX='.prog', OS_TARGET='WINNT', )) return BuildReader(config)
def reader(self, name, enable_tests=False, extra_substs=None): substs = dict( ENABLE_TESTS='1' if enable_tests else '', BIN_SUFFIX='.prog', OS_TARGET='WINNT', COMPILE_ENVIRONMENT='1', ) if extra_substs: substs.update(extra_substs) config = MockConfig(mozpath.join(data_path, name), extra_substs=substs) return BuildReader(config)
def test_filesystem_traversal_reading(self): """Reading moz.build according to filesystem traversal works. We attempt to read every known moz.build file via filesystem traversal. If this test fails, it means that metadata extraction will fail. """ mb = MozbuildObject.from_environment(detect_virtualenv_mozinfo=False) config = mb.config_environment reader = BuildReader(config) all_paths = self._mozbuilds(reader) paths, contexts = reader.read_relevant_mozbuilds(all_paths) self.assertEqual(set(paths), all_paths) self.assertGreaterEqual(len(contexts), len(paths))
def file_info_schedules(self, paths): """Show what is scheduled by the given files. Given a requested set of files (which can be specified using wildcards), print the total set of scheduled components. """ from mozbuild.frontend.reader import EmptyConfig, BuildReader config = EmptyConfig(TOPSRCDIR) reader = BuildReader(config) schedules = set() for p, m in reader.files_info(paths).items(): schedules |= set(m['SCHEDULES'].components) print(", ".join(schedules))
def test_filesystem_traversal_no_config(self): """Reading moz.build files via filesystem traversal mode with no build config. This is similar to the above test except no build config is applied. This will likely fail in more scenarios than the above test because a lot of moz.build files assumes certain variables are present. """ here = os.path.abspath(os.path.dirname(__file__)) root = os.path.normpath(os.path.join(here, '..', '..')) config = EmptyConfig(root) reader = BuildReader(config) all_paths = self._mozbuilds(reader) paths, contexts = reader.read_relevant_mozbuilds(all_paths) self.assertEqual(set(paths.keys()), all_paths) self.assertGreaterEqual(len(contexts), len(paths))
def gen_test_backend(): build_obj = MozbuildObject.from_environment() try: config = build_obj.config_environment except BuildEnvironmentNotFoundException: print("No build detected, test metadata may be incomplete.") config = EmptyConfig(build_obj.topsrcdir) config.topobjdir = build_obj.topobjdir reader = BuildReader(config) emitter = TreeMetadataEmitter(config) backend = TestManifestBackend(config) context = reader.read_topsrcdir() data = emitter.emit(context, emitfn=emitter._process_test_manifests) backend.consume(data)
def test_mtime_no_change(self): """Ensure mtime is not updated if file content does not change.""" env = self._consume('stub0', RecursiveMakeBackend) makefile_path = mozpath.join(env.topobjdir, 'Makefile') backend_path = mozpath.join(env.topobjdir, 'backend.mk') makefile_mtime = os.path.getmtime(makefile_path) backend_mtime = os.path.getmtime(backend_path) reader = BuildReader(env) emitter = TreeMetadataEmitter(env) backend = RecursiveMakeBackend(env) backend.consume(emitter.emit(reader.read_topsrcdir())) self.assertEqual(os.path.getmtime(makefile_path), makefile_mtime) self.assertEqual(os.path.getmtime(backend_path), backend_mtime)
def read_build_config(docdir): """Read the active build config and return the relevant doc paths. The return value is cached so re-generating with the same docdir won't invoke the build system a second time.""" trees = {} python_package_dirs = set() is_main = docdir == MAIN_DOC_PATH relevant_mozbuild_path = None if is_main else docdir # Reading the Sphinx variables doesn't require a full build context. # Only define the parts we need. class fakeconfig(object): topsrcdir = build.topsrcdir variables = ("SPHINX_TREES", "SPHINX_PYTHON_PACKAGE_DIRS") reader = BuildReader(fakeconfig()) result = reader.find_variables_from_ast(variables, path=relevant_mozbuild_path) for path, name, key, value in result: reldir = os.path.dirname(path) if name == "SPHINX_TREES": # If we're building a subtree, only process that specific subtree. # topsrcdir always uses POSIX-style path, normalize it for proper comparison. absdir = os.path.normpath( os.path.join(build.topsrcdir, reldir, value)) if not is_main and absdir not in (docdir, MAIN_DOC_PATH): continue assert key if key.startswith("/"): key = key[1:] else: key = os.path.normpath(os.path.join(reldir, key)) if key in trees: raise Exception( "%s has already been registered as a destination." % key) trees[key] = os.path.join(reldir, value) if name == "SPHINX_PYTHON_PACKAGE_DIRS": python_package_dirs.add(os.path.join(reldir, value)) return trees, python_package_dirs
def resolver(request, tmpdir, topsrcdir, all_tests, defaults): topobjdir = tmpdir.mkdir("objdir").strpath loader_cls = request.param if loader_cls == BuildBackendLoader: with open(os.path.join(topobjdir, 'all-tests.pkl'), 'wb') as fh: pickle.dump(all_tests, fh) with open(os.path.join(topobjdir, 'test-defaults.pkl'), 'wb') as fh: pickle.dump(defaults, fh) resolver = TestResolver(topsrcdir, None, None, topobjdir=topobjdir, loader_cls=loader_cls) resolver._puppeteer_loaded = True resolver._wpt_loaded = True if loader_cls == TestManifestLoader: config = MockConfig(topsrcdir) resolver.load_tests.reader = BuildReader(config) return resolver
def get_outgoing_metadata(self): paths, tags, flavors = set(), set(), set() changed_files = self.vcs.get_outgoing_files('AM') if changed_files: config = EmptyConfig(self.topsrcdir) reader = BuildReader(config) files_info = reader.files_info(changed_files) for path, info in files_info.items(): paths |= info.test_files tags |= info.test_tags flavors |= info.test_flavors return { 'paths': paths, 'tags': tags, 'flavors': flavors, }
def do_test_backend(self, *backends, **kwargs): topobjdir = mkdtemp() try: config = ConfigEnvironment(buildconfig.topsrcdir, topobjdir, **kwargs) reader = BuildReader(config) emitter = TreeMetadataEmitter(config) moz_build = mozpath.join(config.topsrcdir, 'test.mozbuild') definitions = list(emitter.emit( reader.read_mozbuild(moz_build, config))) for backend in backends: backend(config).consume(definitions) yield config except: raise finally: if not os.environ.get('MOZ_NO_CLEANUP'): shutil.rmtree(topobjdir)
def do_test_backend(self, *backends, **kwargs): # Create the objdir in the srcdir to ensure that they share # the same drive on Windows. topobjdir = mkdtemp(dir=buildconfig.topsrcdir) try: config = ConfigEnvironment(buildconfig.topsrcdir, topobjdir, **kwargs) reader = BuildReader(config) emitter = TreeMetadataEmitter(config) moz_build = mozpath.join(config.topsrcdir, "test.mozbuild") definitions = list( emitter.emit(reader.read_mozbuild(moz_build, config))) for backend in backends: backend(config).consume(definitions) yield config except Exception: raise finally: if not os.environ.get("MOZ_NO_CLEANUP"): shutil.rmtree(topobjdir)
def read_build_config(docdir): """Read the active build config and return the relevant doc paths. The return value is cached so re-generating with the same docdir won't invoke the build system a second time.""" trees = {} python_package_dirs = set() is_main = docdir == MAIN_DOC_PATH relevant_mozbuild_path = None if is_main else docdir # Reading the Sphinx variables doesn't require a full build context. # Only define the parts we need. class fakeconfig(object): topsrcdir = build.topsrcdir reader = BuildReader(fakeconfig()) for path, name, key, value in reader.find_sphinx_variables(relevant_mozbuild_path): reldir = os.path.join(os.path.dirname(path), value) if name == 'SPHINX_TREES': # If we're building a subtree, only process that specific subtree. absdir = os.path.join(build.topsrcdir, reldir) if not is_main and absdir not in (docdir, MAIN_DOC_PATH): continue assert key if key.startswith('/'): key = key[1:] else: key = os.path.join(reldir, key) if key in trees: raise Exception('%s has already been registered as a destination.' % key) trees[key] = reldir if name == 'SPHINX_PYTHON_PACKAGE_DIRS': python_package_dirs.add(reldir) return trees, python_package_dirs
def test_orphan_file_patterns(self): if sys.platform == 'win32': raise unittest.SkipTest('failing on windows builds') mb = MozbuildObject.from_environment(detect_virtualenv_mozinfo=False) try: config = mb.config_environment except Exception as e: if e.message == 'config.status not available. Run configure.': raise unittest.SkipTest('failing without config.status') raise reader = BuildReader(config) all_paths = self._mozbuilds(reader) _, contexts = reader.read_relevant_mozbuilds(all_paths) finder = FileFinder(config.topsrcdir, find_executables=False, ignore=['obj*']) def pattern_exists(pat): return [p for p in finder.find(pat)] != [] for ctx in contexts: if not isinstance(ctx, Files): continue relsrcdir = ctx.relsrcdir if not pattern_exists(os.path.join(relsrcdir, ctx.pattern)): self.fail("The pattern '%s' in a Files() entry in " "'%s' corresponds to no files in the tree.\n" "Please update this entry." % (ctx.pattern, ctx.main_path)) test_files = ctx['IMPACTED_TESTS'].files for p in test_files: if not pattern_exists( os.path.relpath(p.full_path, config.topsrcdir)): self.fail("The pattern '%s' in a dependent tests entry " "in '%s' corresponds to no files in the tree.\n" "Please update this entry." % (p, ctx.main_path))
def build_docs(self, format=None, outdir=None): self._activate_virtualenv() self.virtualenv_manager.install_pip_package('sphinx_rtd_theme==0.1.6') from moztreedocs import SphinxManager if outdir == '<DEFAULT>': outdir = os.path.join(self.topobjdir, 'docs') manager = SphinxManager(self.topsrcdir, os.path.join(self.topsrcdir, 'tools', 'docs'), outdir) # We don't care about GYP projects, so don't process them. This makes # scanning faster and may even prevent an exception. def remove_gyp_dirs(context): context['GYP_DIRS'][:] = [] # Reading the Sphinx variables doesn't require a full build context. # Only define the parts we need. class fakeconfig(object): def __init__(self, topsrcdir): self.topsrcdir = topsrcdir config = fakeconfig(self.topsrcdir) reader = BuildReader(config) for path, name, key, value in reader.find_sphinx_variables(): reldir = os.path.dirname(path) if name == 'SPHINX_TREES': assert key manager.add_tree(os.path.join(reldir, value), os.path.join(reldir, key)) if name == 'SPHINX_PYTHON_PACKAGE_DIRS': manager.add_python_package_dir(os.path.join(reldir, value)) return manager.generate_docs(format)
def gen_test_backend(): build_obj = MozbuildObject.from_environment() try: config = build_obj.config_environment except BuildEnvironmentNotFoundException: print("No build detected, test metadata may be incomplete.") # If 'JS_STANDALONE' is set, tests that don't require an objdir won't # be picked up due to bug 1345209. substs = EmptyConfig.default_substs if 'JS_STANDALONE' in substs: del substs['JS_STANDALONE'] config = EmptyConfig(build_obj.topsrcdir, substs) config.topobjdir = build_obj.topobjdir reader = BuildReader(config) emitter = TreeMetadataEmitter(config) backend = TestManifestBackend(config) context = reader.read_topsrcdir() data = emitter.emit(context, emitfn=emitter._process_test_manifests) backend.consume(data)