def test_serialization(self): m = self._get_test_manifest() p = self.tmppath('m') m.write(path=p) self.assertTrue(os.path.isfile(p)) with open(p, 'rb') as fh: c = fh.read() self.assertEqual(c.count('\n'), 5) lines = c.splitlines() self.assertEqual(len(lines), 5) self.assertEqual(lines[0], '2') self.assertEqual(lines[1], '2\x1fc_dest\x1f%s' % self.tmppath('c_source')) self.assertEqual(lines[2], '3\x1fe_dest') self.assertEqual(lines[3], '4\x1fo_dest') self.assertEqual(lines[4], '1\x1fs_dest\x1f%s' % self.tmppath('s_source')) m2 = InstallManifest(path=p) self.assertEqual(m, m2) p2 = self.tmppath('m2') m2.write(path=p2) with open(p2, 'rb') as fh: c2 = fh.read() self.assertEqual(c, c2)
def test_serialization(self): m = self._get_test_manifest() p = self.tmppath("m") m.write(path=p) self.assertTrue(os.path.isfile(p)) with open(p, "rb") as fh: c = fh.read() self.assertEqual(c.count("\n"), 4) lines = c.splitlines() self.assertEqual(len(lines), 4) self.assertEqual(lines[0], "1") self.assertEqual(lines[1], "2\x1fc_dest\x1f%s" % self.tmppath("c_source")) self.assertEqual(lines[2], "3\x1fe_dest") self.assertEqual(lines[3], "1\x1fs_dest\x1f%s" % self.tmppath("s_source")) m2 = InstallManifest(path=p) self.assertEqual(m, m2) p2 = self.tmppath("m2") m2.write(path=p2) with open(p2, "rb") as fh: c2 = fh.read() self.assertEqual(c, c2)
def test_serialization(self): m = self._get_test_manifest() p = self.tmppath('m') m.write(path=p) self.assertTrue(os.path.isfile(p)) with open(p, 'rb') as fh: c = fh.read() self.assertEqual(c.count('\n'), 9) lines = c.splitlines() self.assertEqual(len(lines), 9) self.assertEqual(lines[0], '5') m2 = InstallManifest(path=p) self.assertEqual(m, m2) p2 = self.tmppath('m2') m2.write(path=p2) with open(p2, 'rb') as fh: c2 = fh.read() self.assertEqual(c, c2)
def test_serialization(self): m = self._get_test_manifest() p = self.tmppath("m") m.write(path=p) self.assertTrue(os.path.isfile(p)) with open(p, "rb") as fh: c = fh.read() self.assertEqual(c.count("\n"), 8) lines = c.splitlines() self.assertEqual(len(lines), 8) self.assertEqual(lines[0], "4") m2 = InstallManifest(path=p) self.assertEqual(m, m2) p2 = self.tmppath("m2") m2.write(path=p2) with open(p2, "rb") as fh: c2 = fh.read() self.assertEqual(c, c2)
def process_manifest(destdir, paths, remove_unaccounted=True): manifest = InstallManifest() for path in paths: manifest |= InstallManifest(path=path) copier = FileCopier() manifest.populate_registry(copier) return copier.copy(destdir, remove_unaccounted=remove_unaccounted)
def process_manifest(destdir, *paths): manifest = InstallManifest() for path in paths: manifest |= InstallManifest(path=path) copier = FileCopier() manifest.populate_registry(copier) return copier.copy(destdir)
def test_final_target_files_wildcard(self): """Ensure that wildcards in FINAL_TARGET_FILES work properly.""" env = self._consume('final-target-files-wildcard', FasterMakeBackend) m = InstallManifest(path=mozpath.join(env.topobjdir, 'faster', 'install_dist_bin')) self.assertEqual(len(m), 1) reg = FileRegistry() m.populate_registry(reg) expected = [('foo/bar.xyz', 'bar.xyz'), ('foo/foo.xyz', 'foo.xyz')] actual = [(path, mozpath.relpath(f.path, env.topsrcdir)) for (path, f) in reg] self.assertEqual(expected, actual)
def _get_test_manifest(self): m = InstallManifest() m.add_symlink(self.tmppath('s_source'), 's_dest') m.add_copy(self.tmppath('c_source'), 'c_dest') m.add_required_exists('e_dest') m.add_optional_exists('o_dest') return m
def file_copier(self): # TODO: invalidate the file copier when the build system # itself changes, i.e., the underlying unified manifest # changes. file_copier = FileCopier() unified_manifest = InstallManifest( mozpath.join(self.config_environment.topobjdir, 'faster', 'unified_install_dist_bin')) unified_manifest.populate_registry(file_copier, defines_override=self.defines) return file_copier
def test_old_install_manifest_deleted(self): # Simulate an install manifest from a previous backend version. Ensure # it is deleted. env = self._get_environment('stub0') purge_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install') manifest_path = mozpath.join(purge_dir, 'old_manifest') os.makedirs(purge_dir) m = InstallManifest() m.write(path=manifest_path) self.assertTrue(os.path.exists(manifest_path)) self._consume('stub0', RecursiveMakeBackend, env) self.assertFalse(os.path.exists(manifest_path))
def test_or(self): m1 = self._get_test_manifest() m2 = InstallManifest() m2.add_symlink("s_source2", "s_dest2") m2.add_copy("c_source2", "c_dest2") m1 |= m2 self.assertEqual(len(m2), 2) self.assertEqual(len(m1), 5) self.assertIn("s_dest2", m1) self.assertIn("c_dest2", m1)
def process_manifest(destdir, paths, remove_unaccounted=True, remove_all_directory_symlinks=True, remove_empty_directories=True): manifest = InstallManifest() for path in paths: manifest |= InstallManifest(path=path) copier = FileCopier() manifest.populate_registry(copier) return copier.copy(destdir, remove_unaccounted=remove_unaccounted, remove_all_directory_symlinks=remove_all_directory_symlinks, remove_empty_directories=remove_empty_directories)
def find_generated_harness_files(): # TEST_HARNESS_FILES end up in an install manifest at # $topsrcdir/_build_manifests/install/_tests. manifest = InstallManifest(mozpath.join(buildconfig.topobjdir, '_build_manifests', 'install', '_tests')) registry = FileRegistry() manifest.populate_registry(registry) # Conveniently, the generated files we care about will already # exist in the objdir, so we can identify relevant files if # they're an `ExistingFile` instance. return [mozpath.join('_tests', p) for p in registry.paths() if isinstance(registry[p], ExistingFile)]
def _init(self): CommonBackend._init(self) self._backend_files = {} self._ipdl_sources = set() def detailed(summary): return '{:d} total backend files. {:d} created; {:d} updated; {:d} unchanged'.format( summary.managed_count, summary.created_count, summary.updated_count, summary.unchanged_count) # This is a little kludgy and could be improved with a better API. self.summary.backend_detailed_summary = types.MethodType( detailed, self.summary) self.xpcshell_manifests = [] self.backend_input_files.add( os.path.join(self.environment.topobjdir, 'config', 'autoconf.mk')) self._install_manifests = dict() self._purge_manifests = dict( dist_bin=PurgeManifest(relpath='dist/bin'), dist_include=PurgeManifest(relpath='dist/include'), dist_private=PurgeManifest(relpath='dist/private'), dist_public=PurgeManifest(relpath='dist/public'), dist_sdk=PurgeManifest(relpath='dist/sdk'), tests=PurgeManifest(relpath='_tests'), xpidl=PurgeManifest(relpath='config/makefiles/xpidl'), ) self._install_manifests = dict(dist_idl=InstallManifest(), )
def _init(self): CommonBackend._init(self) self._backend_files = {} self._ipdl_sources = set() self._webidl_sources = set() self._generated_events_webidl_sources = set() self._test_webidl_sources = set() self._preprocessed_test_webidl_sources = set() self._preprocessed_webidl_sources = set() self._generated_webidl_sources = set() def detailed(summary): s = '{:d} total backend files. {:d} created; {:d} updated; {:d} unchanged'.format( summary.created_count + summary.updated_count + summary.unchanged_count, summary.created_count, summary.updated_count, summary.unchanged_count) if summary.deleted_count: s += '; {:d} deleted'.format(summary.deleted_count) return s # This is a little kludgy and could be improved with a better API. self.summary.backend_detailed_summary = types.MethodType( detailed, self.summary) self._test_manifests = {} self.backend_input_files.add( os.path.join(self.environment.topobjdir, 'config', 'autoconf.mk')) self._install_manifests = { k: InstallManifest() for k in [ 'dist_bin', 'dist_idl', 'dist_include', 'dist_public', 'dist_private', 'dist_sdk', 'tests', 'xpidl', ] } self._traversal = RecursiveMakeTraversal() self._may_skip = { 'export': set(), 'compile': set(), 'binaries': set(), 'libs': set(), 'tools': set(), } derecurse = self.environment.substs.get('MOZ_PSEUDO_DERECURSE', '').split(',') self._parallel_export = False self._no_skip = False if derecurse != ['']: self._parallel_export = 'no-parallel-export' not in derecurse self._no_skip = 'no-skip' in derecurse
def find_generated_harness_files(): # TEST_HARNESS_FILES end up in an install manifest at # $topsrcdir/_build_manifests/install/_tests. manifest = InstallManifest( mozpath.join(buildconfig.topobjdir, "_build_manifests", "install", "_tests") ) registry = FileRegistry() manifest.populate_registry(registry) # Conveniently, the generated files we care about will already # exist in the objdir, so we can identify relevant files if # they're an `ExistingFile` instance. return [ mozpath.join("_tests", p) for p in registry.paths() if isinstance(registry[p], ExistingFile) ]
def test_old_install_manifest_deleted(self): # Simulate an install manifest from a previous backend version. Ensure # it is deleted. env = self._get_environment('stub0') purge_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install') manifest_path = mozpath.join(purge_dir, 'old_manifest') os.makedirs(purge_dir) m = InstallManifest() m.write(path=manifest_path) with open(mozpath.join(env.topobjdir, 'backend.RecursiveMakeBackend'), 'w') as f: f.write('%s\n' % manifest_path) self.assertTrue(os.path.exists(manifest_path)) self._consume('stub0', RecursiveMakeBackend, env) self.assertFalse(os.path.exists(manifest_path))
def process_manifest(destdir, paths, remove_unaccounted=True, remove_all_directory_symlinks=True, remove_empty_directories=True): manifest = InstallManifest() for path in paths: manifest |= InstallManifest(path=path) copier = FileCopier() manifest.populate_registry(copier) return copier.copy( destdir, remove_unaccounted=remove_unaccounted, remove_all_directory_symlinks=remove_all_directory_symlinks, remove_empty_directories=remove_empty_directories)
def test_install_manifests_package_tests(self): """Ensure test suites honor package_tests=False.""" env = self._consume('test-manifests-package-tests', RecursiveMakeBackend) tests_dir = mozpath.join(env.topobjdir, '_tests') all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json') self.assertTrue(os.path.exists(all_tests_path)) with open(all_tests_path, 'rt') as fh: o = json.load(fh) self.assertIn('mochitest.js', o) self.assertIn('not_packaged.java', o) man_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install') self.assertTrue(os.path.isdir(man_dir)) full = mozpath.join(man_dir, '_tests') self.assertTrue(os.path.exists(full)) m = InstallManifest(path=full) # Only mochitest.js should be in the install manifest. self.assertTrue('testing/mochitest/tests/mochitest.js' in m) # The path is odd here because we do not normalize at test manifest # processing time. This is a fragile test because there's currently no # way to iterate the manifest. self.assertFalse('instrumentation/./not_packaged.java' in m)
def assertNotInManifest(self, project_name, *args): manifest_path = mozpath.join(self.env.topobjdir, 'android_eclipse', '%s.manifest' % project_name) manifest = InstallManifest(manifest_path) for arg in args: self.assertNotIn( arg, manifest, '%s not in manifest for project %s' % (arg, project_name))
def describe_install_manifest(manifest, dest_dir): try: manifest = InstallManifest(manifest) except UnreadableInstallManifest: raise IOError(errno.EINVAL, 'Error parsing manifest file', manifest) reg = FileRegistry() mapping = {} manifest.populate_registry(reg) for dest_file, src in reg: if hasattr(src, 'path'): dest_path = mozpath.join(dest_dir, dest_file) relsrc_path = mozpath.relpath(src.path, buildconfig.topsrcdir) mapping[dest_path] = relsrc_path return mapping
def test_test_manifests_duplicate_support_files(self): """Ensure duplicate support-files in test manifests work.""" env = self._consume('test-manifests-duplicate-support-files', RecursiveMakeBackend) p = os.path.join(env.topobjdir, '_build_manifests', 'install', 'tests') m = InstallManifest(p) self.assertIn('testing/mochitest/tests/support-file.txt', m)
def _synchronize_docs(self): m = InstallManifest() m.add_symlink(self._conf_py_path, 'conf.py') for dest, source in sorted(self._trees.items()): source_dir = os.path.join(self._topsrcdir, source) for root, dirs, files in os.walk(source_dir): for f in files: source_path = os.path.join(root, f) rel_source = source_path[len(source_dir) + 1:] m.add_symlink(source_path, os.path.join(dest, rel_source)) stage_dir = os.path.join(self._output_dir, 'staging') copier = FileCopier() m.populate_registry(copier) copier.copy(stage_dir) with open(self._index_path, 'rb') as fh: data = fh.read() indexes = ['%s/index' % p for p in sorted(self._trees.keys())] indexes = '\n '.join(indexes) packages = [os.path.basename(p) for p in self._python_package_dirs] packages = ['python/%s' % p for p in packages] packages = '\n '.join(sorted(packages)) data = data.format(indexes=indexes, python_packages=packages) with open(os.path.join(stage_dir, 'index.rst'), 'wb') as fh: fh.write(data)
def _synchronize_docs(self): m = InstallManifest() m.add_symlink(self._conf_py_path, "conf.py") for dest, source in sorted(self._trees.items()): source_dir = os.path.join(self._topsrcdir, source) for root, dirs, files in os.walk(source_dir): for f in files: source_path = os.path.join(root, f) rel_source = source_path[len(source_dir) + 1 :] m.add_symlink(source_path, os.path.join(dest, rel_source)) copier = FileCopier() m.populate_registry(copier) copier.copy(self._docs_dir) with open(self._index_path, "rb") as fh: data = fh.read() indexes = ["%s/index" % p for p in sorted(self._trees.keys())] indexes = "\n ".join(indexes) packages = [os.path.basename(p) for p in self._python_package_dirs] packages = ["python/%s" % p for p in packages] packages = "\n ".join(sorted(packages)) data = data.format(indexes=indexes, python_packages=packages) with open(os.path.join(self._docs_dir, "index.rst"), "wb") as fh: fh.write(data)
def _get_test_manifest(self): m = InstallManifest() m.add_symlink(self.tmppath("s_source"), "s_dest") m.add_copy(self.tmppath("c_source"), "c_dest") m.add_required_exists("e_dest") return m
def test_test_support_files_tracked(self): env = self._consume('test-support-binaries-tracked', RecursiveMakeBackend) m = InstallManifest(path=mozpath.join(env.topobjdir, '_build_manifests', 'install', '_tests')) self.assertEqual(len(m), 4) self.assertIn('xpcshell/tests/mozbuildtest/test-library.dll', m) self.assertIn('xpcshell/tests/mozbuildtest/test-one.exe', m) self.assertIn('xpcshell/tests/mozbuildtest/test-two.exe', m) self.assertIn('xpcshell/tests/mozbuildtest/host-test-library.dll', m)
def test_pattern_expansion(self): source = self.tmppath("source") os.mkdir(source) os.mkdir("%s/base" % source) os.mkdir("%s/base/foo" % source) with open("%s/base/foo/file1" % source, "a"): pass with open("%s/base/foo/file2" % source, "a"): pass m = InstallManifest() m.add_pattern_symlink("%s/base" % source, "**", "dest") c = FileCopier() m.populate_registry(c) self.assertEqual(c.paths(), ["dest/foo/file1", "dest/foo/file2"])
def test_pattern_expansion(self): source = self.tmppath('source') os.mkdir(source) os.mkdir('%s/base' % source) os.mkdir('%s/base/foo' % source) with open('%s/base/foo/file1' % source, 'a'): pass with open('%s/base/foo/file2' % source, 'a'): pass m = InstallManifest() m.add_pattern_symlink('%s/base' % source, '**', 'dest') c = FileCopier() m.populate_registry(c) self.assertEqual(c.paths(), ['dest/foo/file1', 'dest/foo/file2'])
def describe_install_manifest(manifest, dest_dir): try: manifest = InstallManifest(manifest) except UnreadableInstallManifest: raise IOError(errno.EINVAL, 'Error parsing manifest file', manifest) reg = FileRegistry() mapping = {} manifest.populate_registry(reg) dest_dir = mozpath.join(buildconfig.topobjdir, dest_dir) for dest_file, src in reg: if hasattr(src, 'path'): dest_path = mozpath.join(dest_dir, dest_file) relsrc_path = mozpath.relpath(src.path, buildconfig.topsrcdir) mapping[dest_path] = relsrc_path return mapping
def consume_finished(self): mp = os.path.join(self.environment.topobjdir, '_build_manifests', 'install', '_tests') install_manifest = InstallManifest(mp) reg = FileRegistry() install_manifest.populate_registry(reg) for dest, src in reg: if not hasattr(src, 'path'): continue if not os.path.isabs(dest): dest = '_tests/' + dest obj_path = mozpath.join(self.environment.topobjdir, dest) if isinstance(src, PreprocessedFile): assert os.path.exists(obj_path), '%s should exist' % obj_path pp_info = generate_pp_info(obj_path, self.environment.topsrcdir) else: pp_info = None self._install_mapping[dest] = src.path, pp_info # Our result has four parts: # A map from url prefixes to objdir directories: # { "chrome://mozapps/content/": [ "dist/bin/chrome/toolkit/content/mozapps" ], ... } # A map of overrides. # A map from objdir paths to sourcedir paths, and an object storing mapping information for preprocessed files: # { "dist/bin/browser/chrome/browser/content/browser/aboutSessionRestore.js": # [ "$topsrcdir/browser/components/sessionstore/content/aboutSessionRestore.js", {} ], ... } # An object containing build configuration information. outputfile = os.path.join(self.environment.topobjdir, 'chrome-map.json') with self._write_file(outputfile) as fh: chrome_mapping = self.manifest_handler.chrome_mapping overrides = self.manifest_handler.overrides json.dump([ {k: list(v) for k, v in chrome_mapping.iteritems()}, overrides, self._install_mapping, { 'topobjdir': mozpath.normpath(self.environment.topobjdir), 'MOZ_APP_NAME': self.environment.substs.get('MOZ_APP_NAME'), 'OMNIJAR_NAME': self.environment.substs.get('OMNIJAR_NAME'), 'MOZ_MACBUNDLE_NAME': self.environment.substs.get('MOZ_MACBUNDLE_NAME'), } ], fh, sort_keys=True, indent=2)
def test_install_manifests_written(self): env, objs = self._emit("stub0") backend = RecursiveMakeBackend(env) m = InstallManifest() backend._install_manifests["testing"] = m m.add_symlink(__file__, "self") backend.consume(objs) man_dir = os.path.join(env.topobjdir, "_build_manifests", "install") self.assertTrue(os.path.isdir(man_dir)) expected = ["testing"] for e in expected: full = os.path.join(man_dir, e) self.assertTrue(os.path.exists(full)) m2 = InstallManifest(path=full) self.assertEqual(m, m2)
def process_manifest(destdir, paths, track=None, remove_unaccounted=True, remove_all_directory_symlinks=True, remove_empty_directories=True, defines={}): if track: if os.path.exists(track): # We use the same format as install manifests for the tracking # data. manifest = InstallManifest(path=track) remove_unaccounted = FileRegistry() dummy_file = BaseFile() finder = FileFinder(destdir, find_executables=False, find_dotfiles=True) for dest in manifest._dests: if '*' in dest: for p, f in finder.find(dest): remove_unaccounted.add(p, dummy_file) else: remove_unaccounted.add(dest, dummy_file) else: # If tracking is enabled and there is no file, we don't want to # be removing anything. remove_unaccounted = False remove_empty_directories = False remove_all_directory_symlinks = False manifest = InstallManifest() for path in paths: manifest |= InstallManifest(path=path) copier = FileCopier() manifest.populate_registry(copier, defines_override=defines) result = copier.copy( destdir, remove_unaccounted=remove_unaccounted, remove_all_directory_symlinks=remove_all_directory_symlinks, remove_empty_directories=remove_empty_directories) if track: manifest.write(path=track) return result
def process_manifest(destdir, paths, track=None, remove_unaccounted=True, remove_all_directory_symlinks=True, remove_empty_directories=True, no_symlinks=False, defines={}): if track: if os.path.exists(track): # We use the same format as install manifests for the tracking # data. manifest = InstallManifest(path=track) remove_unaccounted = FileRegistry() dummy_file = BaseFile() finder = FileFinder(destdir, find_dotfiles=True) for dest in manifest._dests: for p, f in finder.find(dest): remove_unaccounted.add(p, dummy_file) else: # If tracking is enabled and there is no file, we don't want to # be removing anything. remove_unaccounted=False remove_empty_directories=False remove_all_directory_symlinks=False manifest_cls = InstallManifestNoSymlinks if no_symlinks else InstallManifest manifest = manifest_cls() for path in paths: manifest |= manifest_cls(path=path) copier = FileCopier() manifest.populate_registry(copier, defines_override=defines) result = copier.copy(destdir, remove_unaccounted=remove_unaccounted, remove_all_directory_symlinks=remove_all_directory_symlinks, remove_empty_directories=remove_empty_directories) if track: manifest.write(path=track) return result
def test_install_manifests_written(self): env, objs = self._emit('stub0') backend = RecursiveMakeBackend(env) m = InstallManifest() backend._install_manifests['testing'] = m m.add_symlink(__file__, 'self') backend.consume(objs) man_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install') self.assertTrue(os.path.isdir(man_dir)) expected = ['testing'] for e in expected: full = mozpath.join(man_dir, e) self.assertTrue(os.path.exists(full)) m2 = InstallManifest(path=full) self.assertEqual(m, m2)
def install_test_files(topsrcdir, topobjdir, tests_root, test_objs): """Installs the requested test files to the objdir. This is invoked by test runners to avoid installing tens of thousands of test files when only a few tests need to be run. """ flavor_info = {flavor: (root, prefix, install) for (flavor, root, prefix, install) in TEST_MANIFESTS.values()} objdir_dest = mozpath.join(topobjdir, tests_root) converter = SupportFilesConverter() install_info = TestInstallInfo() for o in test_objs: flavor = o['flavor'] if flavor not in flavor_info: # This is a test flavor that isn't installed by the build system. continue root, prefix, install = flavor_info[flavor] if not install: # This flavor isn't installed to the objdir. continue manifest_path = o['manifest'] manifest_dir = mozpath.dirname(manifest_path) out_dir = mozpath.join(root, prefix, manifest_dir[len(topsrcdir) + 1:]) file_relpath = o['file_relpath'] source = mozpath.join(topsrcdir, file_relpath) dest = mozpath.join(root, prefix, file_relpath) if 'install-to-subdir' in o: out_dir = mozpath.join(out_dir, o['install-to-subdir']) manifest_relpath = mozpath.relpath(source, mozpath.dirname(manifest_path)) dest = mozpath.join(out_dir, manifest_relpath) install_info.installs.append((source, dest)) install_info |= converter.convert_support_files(o, root, manifest_dir, out_dir) manifest = InstallManifest() for source, dest in set(install_info.installs): if dest in install_info.external_installs: continue manifest.add_symlink(source, dest) for base, pattern, dest in install_info.pattern_installs: manifest.add_pattern_symlink(base, pattern, dest) _resolve_installs(install_info.deferred_installs, topobjdir, manifest) # Harness files are treated as a monolith and installed each time we run tests. # Fortunately there are not very many. manifest |= InstallManifest(mozpath.join(topobjdir, '_build_manifests', 'install', tests_root)) copier = FileCopier() manifest.populate_registry(copier) copier.copy(objdir_dest, remove_unaccounted=False)
def test_install_manifests_written(self): env, objs = self._emit("stub0") backend = RecursiveMakeBackend(env) m = InstallManifest() backend._install_manifests["testing"] = m m.add_link(__file__, "self") backend.consume(objs) man_dir = mozpath.join(env.topobjdir, "_build_manifests", "install") self.assertTrue(os.path.isdir(man_dir)) expected = ["testing"] for e in expected: full = mozpath.join(man_dir, e) self.assertTrue(os.path.exists(full)) m2 = InstallManifest(path=full) self.assertEqual(m, m2)
def test_process_manifest(self): source = self.tmppath('source') os.mkdir(source) os.mkdir('%s/base' % source) os.mkdir('%s/base/foo' % source) os.mkdir('%s/base2' % source) with open('%s/base/foo/file1' % source, 'a'): pass with open('%s/base/foo/file2' % source, 'a'): pass with open('%s/base2/file3' % source, 'a'): pass m = InstallManifest() m.add_pattern_link('%s/base' % source, '**', '') m.add_link('%s/base2/file3' % source, 'foo/file3') p = self.tmppath('m') m.write(path=p) dest = self.tmppath('dest') track = self.tmppath('track') for i in range(2): process_install_manifest.process_manifest(dest, [p], track) self.assertTrue(os.path.exists(self.tmppath('dest/foo/file1'))) self.assertTrue(os.path.exists(self.tmppath('dest/foo/file2'))) self.assertTrue(os.path.exists(self.tmppath('dest/foo/file3'))) m = InstallManifest() m.write(path=p) for i in range(2): process_install_manifest.process_manifest(dest, [p], track) self.assertFalse(os.path.exists(self.tmppath('dest/foo/file1'))) self.assertFalse(os.path.exists(self.tmppath('dest/foo/file2'))) self.assertFalse(os.path.exists(self.tmppath('dest/foo/file3')))
def test_branding_files(self): """Ensure BRANDING_FILES is handled properly.""" env = self._consume('branding-files', RecursiveMakeBackend) #BRANDING_FILES should appear in the dist_branding install manifest. m = InstallManifest(path=os.path.join( env.topobjdir, '_build_manifests', 'install', 'dist_branding')) self.assertEqual(len(m), 3) self.assertIn('bar.ico', m) self.assertIn('quux.png', m) self.assertIn('icons/foo.ico', m)
def test_exports(self): """Ensure EXPORTS is handled properly.""" env = self._consume('exports', RecursiveMakeBackend) # EXPORTS files should appear in the dist_include install manifest. m = InstallManifest(path=mozpath.join( env.topobjdir, '_build_manifests', 'install', 'dist_include')) self.assertEqual(len(m), 7) self.assertIn('foo.h', m) self.assertIn('mozilla/mozilla1.h', m) self.assertIn('mozilla/dom/dom2.h', m)
def test_test_manifest_pattern_matches_recorded(self): """Pattern matches in test manifests' support-files should be recorded.""" env = self._consume('test-manifests-written', RecursiveMakeBackend) m = InstallManifest(path=mozpath.join( env.topobjdir, '_build_manifests', 'install', 'tests')) # This is not the most robust test in the world, but it gets the job # done. entries = [e for e in m._dests.keys() if '**' in e] self.assertEqual(len(entries), 1) self.assertIn('support/**', entries[0])
def process_manifest(destdir, paths, track, no_symlinks=False, defines={}): if os.path.exists(track): # We use the same format as install manifests for the tracking # data. manifest = InstallManifest(path=track) remove_unaccounted = FileRegistry() dummy_file = BaseFile() finder = FileFinder(destdir, find_dotfiles=True) for dest in manifest._dests: for p, f in finder.find(dest): remove_unaccounted.add(p, dummy_file) remove_empty_directories = True remove_all_directory_symlinks = True else: # If tracking is enabled and there is no file, we don't want to # be removing anything. remove_unaccounted = False remove_empty_directories = False remove_all_directory_symlinks = False manifest = InstallManifest() for path in paths: manifest |= InstallManifest(path=path) copier = FileCopier() link_policy = "copy" if no_symlinks else "symlink" manifest.populate_registry( copier, defines_override=defines, link_policy=link_policy ) result = copier.copy(destdir, remove_unaccounted=remove_unaccounted, remove_all_directory_symlinks=remove_all_directory_symlinks, remove_empty_directories=remove_empty_directories) if track: # We should record files that we actually copied. # It is too late to expand wildcards when the track file is read. manifest.write(path=track, expand_pattern=True) return result
def install_test_files(topsrcdir, topobjdir, tests_root): """Installs the requested test files to the objdir. This is invoked by test runners to avoid installing tens of thousands of test files when only a few tests need to be run. """ manifest = InstallManifest( mozpath.join(topobjdir, '_build_manifests', 'install', '_test_files')) harness_files_manifest = mozpath.join(topobjdir, '_build_manifests', 'install', tests_root) if os.path.isfile(harness_files_manifest): # If the backend has generated an install manifest for test harness # files they are treated as a monolith and installed each time we # run tests. Fortunately there are not very many. manifest |= InstallManifest(harness_files_manifest) copier = FileCopier() manifest.populate_registry(copier) copier.copy(mozpath.join(topobjdir, tests_root), remove_unaccounted=False)
def test_xpidl_generation(self): """Ensure xpidl files and directories are written out.""" env = self._consume("xpidl", RecursiveMakeBackend) # Install manifests should contain entries. install_dir = mozpath.join(env.topobjdir, "_build_manifests", "install") self.assertTrue(os.path.isfile(mozpath.join(install_dir, "xpidl"))) m = InstallManifest(path=mozpath.join(install_dir, "xpidl")) self.assertIn(".deps/my_module.pp", m) m = InstallManifest(path=mozpath.join(install_dir, "xpidl")) self.assertIn("my_module.xpt", m) m = InstallManifest(path=mozpath.join(install_dir, "dist_include")) self.assertIn("foo.h", m) p = mozpath.join(env.topobjdir, "config/makefiles/xpidl") self.assertTrue(os.path.isdir(p)) self.assertTrue(os.path.isfile(mozpath.join(p, "Makefile")))
def test_xpidl_generation(self): """Ensure xpidl files and directories are written out.""" env = self._consume('xpidl', RecursiveMakeBackend) # Install manifests should contain entries. install_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install') self.assertTrue(os.path.isfile(mozpath.join(install_dir, 'xpidl'))) m = InstallManifest(path=mozpath.join(install_dir, 'xpidl')) self.assertIn('.deps/my_module.pp', m) m = InstallManifest(path=mozpath.join(install_dir, 'xpidl')) self.assertIn('my_module.xpt', m) m = InstallManifest(path=mozpath.join(install_dir, 'dist_include')) self.assertIn('foo.h', m) p = mozpath.join(env.topobjdir, 'config/makefiles/xpidl') self.assertTrue(os.path.isdir(p)) self.assertTrue(os.path.isfile(mozpath.join(p, 'Makefile')))
def test_test_manifest_deffered_installs_written(self): """Shared support files are written to their own data file by the backend.""" env = self._consume('test-manifest-shared-support', RecursiveMakeBackend) all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json') self.assertTrue(os.path.exists(all_tests_path)) test_installs_path = mozpath.join(env.topobjdir, 'test-installs.json') with open(test_installs_path, 'r') as fh: test_installs = json.load(fh) self.assertEqual(set(test_installs.keys()), set(['child/test_sub.js', 'child/data/**', 'child/another-file.sjs'])) for key in test_installs.keys(): self.assertIn(key, test_installs) test_files_manifest = mozpath.join(env.topobjdir, '_build_manifests', 'install', '_test_files') # First, read the generated for ini manifest contents. m = InstallManifest(path=test_files_manifest) # Then, synthesize one from the test-installs.json file. This should # allow us to re-create a subset of the above. synthesized_manifest = InstallManifest() for item, installs in test_installs.items(): for install_info in installs: if len(install_info) == 3: synthesized_manifest.add_pattern_symlink(*install_info) if len(install_info) == 2: synthesized_manifest.add_symlink(*install_info) self.assertEqual(len(synthesized_manifest), 3) for item, info in synthesized_manifest._dests.items(): self.assertIn(item, m) self.assertEqual(info, m._dests[item])
def test_exports_generated(self): """Ensure EXPORTS that are listed in GENERATED_FILES are handled properly.""" env = self._consume('exports-generated', RecursiveMakeBackend) # EXPORTS files should appear in the dist_include install manifest. m = InstallManifest(path=mozpath.join( env.topobjdir, '_build_manifests', 'install', 'dist_include')) self.assertEqual(len(m), 8) self.assertIn('foo.h', m) self.assertIn('mozilla/mozilla1.h', m) self.assertIn('mozilla/dom/dom1.h', m) self.assertIn('gfx/gfx.h', m) self.assertIn('bar.h', m) self.assertIn('mozilla/mozilla2.h', m) self.assertIn('mozilla/dom/dom2.h', m) self.assertIn('mozilla/dom/dom3.h', m) # EXPORTS files that are also GENERATED_FILES should be handled as # INSTALL_TARGETS. backend_path = mozpath.join(env.topobjdir, 'backend.mk') lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]] expected = [ 'export:: bar.h', 'GARBAGE += bar.h', 'EXTRA_MDDEPEND_FILES += bar.h.pp', 'export:: mozilla2.h', 'GARBAGE += mozilla2.h', 'EXTRA_MDDEPEND_FILES += mozilla2.h.pp', 'export:: dom2.h', 'GARBAGE += dom2.h', 'EXTRA_MDDEPEND_FILES += dom2.h.pp', 'export:: dom3.h', 'GARBAGE += dom3.h', 'EXTRA_MDDEPEND_FILES += dom3.h.pp', 'dist_include_FILES += bar.h', 'dist_include_DEST := $(DEPTH)/dist/include/', 'dist_include_TARGET := export', 'INSTALL_TARGETS += dist_include', 'dist_include_mozilla_FILES += mozilla2.h', 'dist_include_mozilla_DEST := $(DEPTH)/dist/include/mozilla', 'dist_include_mozilla_TARGET := export', 'INSTALL_TARGETS += dist_include_mozilla', 'dist_include_mozilla_dom_FILES += dom2.h', 'dist_include_mozilla_dom_FILES += dom3.h', 'dist_include_mozilla_dom_DEST := $(DEPTH)/dist/include/mozilla/dom', 'dist_include_mozilla_dom_TARGET := export', 'INSTALL_TARGETS += dist_include_mozilla_dom', ] self.maxDiff = None self.assertEqual(lines, expected)
def setUp(self): HelperMixin.setUp(self) self.srcdir = os.path.join(self.test_dir, 'src') os.mkdir(self.srcdir) self.objdir = os.path.join(self.test_dir, 'obj') os.mkdir(self.objdir) self.manifest = InstallManifest() self.canonical_mapping = {} for s in ['src1', 'src2']: srcfile = os.path.join(self.srcdir, s) objfile = os.path.join(self.objdir, s) self.canonical_mapping[objfile] = srcfile self.manifest.add_copy(srcfile, s) self.manifest_file = os.path.join(self.test_dir, 'install-manifest') self.manifest.write(self.manifest_file)
def _synchronize_docs(self): m = InstallManifest() m.add_link(self.conf_py_path, 'conf.py') for dest, source in sorted(self.trees.items()): source_dir = os.path.join(self.topsrcdir, source) for root, dirs, files in os.walk(source_dir): for f in files: source_path = os.path.join(root, f) rel_source = source_path[len(source_dir) + 1:] m.add_link(source_path, os.path.join(dest, rel_source)) copier = FileCopier() m.populate_registry(copier) copier.copy(self.staging_dir) with open(self.index_path, 'rb') as fh: data = fh.read() def is_toplevel(key): """Whether the tree is nested under the toplevel index, or is nested under another tree's index. """ for k in self.trees: if k == key: continue if key.startswith(k): return False return True toplevel_trees = { k: v for k, v in self.trees.items() if is_toplevel(k) } indexes = ['%s/index' % p for p in sorted(toplevel_trees.keys())] indexes = '\n '.join(indexes) packages = [os.path.basename(p) for p in self.python_package_dirs] packages = ['python/%s' % p for p in packages] packages = '\n '.join(sorted(packages)) data = data.format(indexes=indexes, python_packages=packages) with open(os.path.join(self.staging_dir, 'index.rst'), 'wb') as fh: fh.write(data)
def setUp(self): HelperMixin.setUp(self) self.srcdir = os.path.join(self.test_dir, "src") os.mkdir(self.srcdir) self.objdir = os.path.join(self.test_dir, "obj") os.mkdir(self.objdir) self.manifest = InstallManifest() self.canonical_mapping = {} for s in ["src1", "src2"]: srcfile = os.path.join(self.srcdir, s) objfile = os.path.join(self.objdir, s) self.canonical_mapping[objfile] = srcfile self.manifest.add_copy(srcfile, s) self.manifest_file = os.path.join(self.test_dir, "install-manifest") self.manifest.write(self.manifest_file)
def test_resources(self): """Ensure RESOURCE_FILES is handled properly.""" env = self._consume('resources', RecursiveMakeBackend) # RESOURCE_FILES should appear in the dist_bin install manifest. m = InstallManifest(path=os.path.join( env.topobjdir, '_build_manifests', 'install', 'dist_bin')) self.assertEqual(len(m), 10) self.assertIn('res/foo.res', m) self.assertIn('res/fonts/font1.ttf', m) self.assertIn('res/fonts/desktop/desktop2.ttf', m) self.assertIn('res/bar.res', m) self.assertIn('res/tests/test.manifest', m) self.assertIn('res/tests/extra.manifest', m)
def test_or(self): m1 = self._get_test_manifest() m2 = InstallManifest() m2.add_symlink('s_source2', 's_dest2') m2.add_copy('c_source2', 'c_dest2') m1 |= m2 self.assertEqual(len(m2), 2) self.assertEqual(len(m1), 6) self.assertIn('s_dest2', m1) self.assertIn('c_dest2', m1)
def test_write_expand_pattern(self): source = self.tmppath('source') os.mkdir(source) os.mkdir('%s/base' % source) os.mkdir('%s/base/foo' % source) with open('%s/base/foo/file1' % source, 'a'): pass with open('%s/base/foo/file2' % source, 'a'): pass m = InstallManifest() m.add_pattern_link('%s/base' % source, '**', 'dest') track = self.tmppath('track') m.write(path=track, expand_pattern=True) m = InstallManifest(path=track) self.assertEqual([dest for dest in m._dests], ['dest/foo/file1', 'dest/foo/file2'])
def test_write_expand_pattern(self): source = self.tmppath("source") os.mkdir(source) os.mkdir("%s/base" % source) os.mkdir("%s/base/foo" % source) with open("%s/base/foo/file1" % source, "a"): pass with open("%s/base/foo/file2" % source, "a"): pass m = InstallManifest() m.add_pattern_link("%s/base" % source, "**", "dest") track = self.tmppath("track") m.write(path=track, expand_pattern=True) m = InstallManifest(path=track) self.assertEqual(sorted(dest for dest in m._dests), ["dest/foo/file1", "dest/foo/file2"])
def test_or(self): m1 = self._get_test_manifest() orig_length = len(m1) m2 = InstallManifest() m2.add_link("s_source2", "s_dest2") m2.add_copy("c_source2", "c_dest2") m1 |= m2 self.assertEqual(len(m2), 2) self.assertEqual(len(m1), orig_length + 2) self.assertIn("s_dest2", m1) self.assertIn("c_dest2", m1)
def _make_install_manifest(topsrcdir, topobjdir, test_objs): flavor_info = { flavor: (root, prefix, install) for (flavor, root, prefix, install) in TEST_MANIFESTS.values() } converter = SupportFilesConverter() install_info = TestInstallInfo() for o in test_objs: flavor = o['flavor'] if flavor not in flavor_info: # This is a test flavor that isn't installed by the build system. continue root, prefix, install = flavor_info[flavor] if not install: # This flavor isn't installed to the objdir. continue manifest_path = o['manifest'] manifest_dir = mozpath.dirname(manifest_path) out_dir = mozpath.join(root, prefix, manifest_dir[len(topsrcdir) + 1:]) file_relpath = o['file_relpath'] source = mozpath.join(topsrcdir, file_relpath) dest = mozpath.join(root, prefix, file_relpath) if 'install-to-subdir' in o: out_dir = mozpath.join(out_dir, o['install-to-subdir']) manifest_relpath = mozpath.relpath(source, mozpath.dirname(manifest_path)) dest = mozpath.join(out_dir, manifest_relpath) install_info.installs.append((source, dest)) install_info |= converter.convert_support_files( o, root, manifest_dir, out_dir) manifest = InstallManifest() for source, dest in set(install_info.installs): if dest in install_info.external_installs: continue manifest.add_link(source, dest) for base, pattern, dest in install_info.pattern_installs: manifest.add_pattern_link(base, pattern, dest) _resolve_installs(install_info.deferred_installs, topobjdir, manifest) return manifest
def test_install_manifests_package_tests(self): """Ensure test suites honor package_tests=False.""" env = self._consume("test-manifests-package-tests", RecursiveMakeBackend) man_dir = mozpath.join(env.topobjdir, "_build_manifests", "install") self.assertTrue(os.path.isdir(man_dir)) full = mozpath.join(man_dir, "_test_files") self.assertTrue(os.path.exists(full)) m = InstallManifest(path=full) # Only mochitest.js should be in the install manifest. self.assertTrue("testing/mochitest/tests/mochitest.js" in m) # The path is odd here because we do not normalize at test manifest # processing time. This is a fragile test because there's currently no # way to iterate the manifest. self.assertFalse("instrumentation/./not_packaged.java" in m)
def test_adds(self): m = InstallManifest() m.add_symlink('s_source', 's_dest') m.add_copy('c_source', 'c_dest') m.add_required_exists('e_dest') m.add_optional_exists('o_dest') self.assertEqual(len(m), 4) self.assertIn('s_dest', m) self.assertIn('c_dest', m) self.assertIn('e_dest', m) self.assertIn('o_dest', m) with self.assertRaises(ValueError): m.add_symlink('s_other', 's_dest') with self.assertRaises(ValueError): m.add_copy('c_other', 'c_dest') with self.assertRaises(ValueError): m.add_required_exists('e_dest') with self.assertRaises(ValueError): m.add_optional_exists('o_dest')