def _add_base(self, base, addon=False): if addon: # Because add_base is always called with parents before children, # all the possible ancestry of `base` is already present in # `_sub_formatter`. parent_base = mozpath.basedir(base, self._sub_formatter.keys()) rel_base = mozpath.relpath(base, parent_base) # If the addon is under a resource directory, package it in the # omnijar. parent_sub_formatter = self._sub_formatter[parent_base] if parent_sub_formatter.is_resource(rel_base): omnijar_sub_formatter = \ parent_sub_formatter._sub_formatter[self._omnijar_name] self._sub_formatter[base] = FlatSubFormatter( FileRegistrySubtree(rel_base, omnijar_sub_formatter.copier)) return JarFormatter._add_base(self, base, addon) else: # Initialize a chrome.manifest next to the omnijar file so that # there's always a chrome.manifest file, even an empty one. path = mozpath.normpath(mozpath.join(base, 'chrome.manifest')) if not self.copier.contains(path): self.copier.add(path, ManifestFile('')) self._sub_formatter[base] = OmniJarSubFormatter( FileRegistrySubtree(base, self.copier), self._omnijar_name, self._compress, self._non_resources)
def add_manifest(self, entry): # Store manifest entries in a single manifest per directory, named # after their parent directory, except for root manifests, all named # chrome.manifest. if entry.base: name = mozpath.basename(entry.base) else: name = 'chrome' path = mozpath.normpath(mozpath.join(entry.base, '%s.manifest' % name)) if not self.copier.contains(path): # Add a reference to the manifest file in the parent manifest, if # the manifest file is not a root manifest. if entry.base: parent = mozpath.dirname(entry.base) relbase = mozpath.basename(entry.base) relpath = mozpath.join(relbase, mozpath.basename(path)) self.add_manifest(Manifest(parent, relpath)) self.copier.add(path, ManifestFile(entry.base)) if isinstance(entry, ManifestChrome): data = self._chrome_db.setdefault(entry.name, {}) entries = data.setdefault(entry.type, []) for e in entries: # Ideally, we'd actually check whether entry.flags are more # specific than e.flags, but in practice the following test # is enough for now. if not entry.flags or e.flags and entry.flags == e.flags: errors.fatal('"%s" overrides "%s"' % (entry, e)) entries.append(entry) self.copier[path].add(entry)
def __init__(self, source, omnijar_name=None, unpack_xpi=True): if isinstance(source, BaseFinder): self._finder = source else: self._finder = FileFinder(source) self.base = self._finder.base self.files = FileRegistry() self.kind = "flat" if omnijar_name: self.omnijar = omnijar_name else: # Can't include globally because of bootstrapping issues. from buildconfig import substs self.omnijar = substs.get("OMNIJAR_NAME", "omni.ja") self.jarlogs = {} self.compressed = False self._unpack_xpi = unpack_xpi jars = set() for p, f in self._finder.find("*"): # Skip the precomplete file, which is generated at packaging time. if p == "precomplete": continue base = mozpath.dirname(p) # If the file matches the omnijar pattern, it is an omnijar. # All the files it contains go in the directory containing the full # pattern. Manifests are merged if there is a corresponding manifest # in the directory. if self._maybe_zip(f) and mozpath.match(p, "**/%s" % self.omnijar): jar = self._open_jar(p, f) if "chrome.manifest" in jar: self.kind = "omni" self._fill_with_jar(p[: -len(self.omnijar) - 1], jar) continue # If the file is a manifest, scan its entries for some referencing # jar: urls. If there are some, the files contained in the jar they # point to, go under a directory named after the jar. if is_manifest(p): m = self.files[p] if self.files.contains(p) else ManifestFile(base) for e in parse_manifest( self.base, p, codecs.getreader("utf-8")(f.open()) ): m.add(self._handle_manifest_entry(e, jars)) if self.files.contains(p): continue f = m # If we're unpacking packed addons and the file is a packed addon, # unpack it under a directory named after the xpi. if self._unpack_xpi and p.endswith(".xpi") and self._maybe_zip(f): self._fill_with_jar(p[:-4], self._open_jar(p, f)) continue if p not in jars: self.files.add(p, f)
def find(self, pattern): for p, f in self._finder.find(pattern): if isinstance(f, ManifestFile): unwanted = [ e for e in f._entries if isinstance(e, Manifest) and e.flags ] if unwanted: f = ManifestFile( f._base, [e for e in f._entries if e not in unwanted]) yield p, f
def _add_base(self, base, addon=False): if addon: JarFormatter._add_base(self, base, addon) else: # Initialize a chrome.manifest next to the omnijar file so that # there's always a chrome.manifest file, even an empty one. path = mozpath.normpath(mozpath.join(base, 'chrome.manifest')) if not self.copier.contains(path): self.copier.add(path, ManifestFile('')) self._sub_formatter[base] = OmniJarSubFormatter( FileRegistrySubtree(base, self.copier), self._omnijar_name, self._compress, self._optimize, self._non_resources)
def _fill_with_omnijar(self, base, jar): for j in jar: path = mozpack.path.join(base, j.filename) if is_manifest(j.filename): m = self.files[path] if self.files.contains(path) \ else ManifestFile(mozpack.path.dirname(path)) for e in parse_manifest(None, path, j): m.add(e) if not self.files.contains(path): self.files.add(path, m) continue else: self.files.add(path, DeflatedFile(j))
def __init__(self, source): if isinstance(source, BaseFinder): self._finder = source else: self._finder = FileFinder(source) self.base = self._finder.base self.files = FileRegistry() self.kind = 'flat' self.omnijar = None self.jarlogs = {} self.optimizedjars = False self.compressed = False jars = set() for p, f in self._finder.find('*'): # Skip the precomplete file, which is generated at packaging time. if p == 'precomplete': continue base = mozpath.dirname(p) # If the file is a zip/jar that is not a .xpi, and contains a # chrome.manifest, it is an omnijar. All the files it contains # go in the directory containing the omnijar. Manifests are merged # if there is a corresponding manifest in the directory. if not p.endswith('.xpi') and self._maybe_zip(f) and \ (mozpath.basename(p) == self.omnijar or not self.omnijar): jar = self._open_jar(p, f) if 'chrome.manifest' in jar: self.kind = 'omni' self.omnijar = mozpath.basename(p) self._fill_with_jar(base, jar) continue # If the file is a manifest, scan its entries for some referencing # jar: urls. If there are some, the files contained in the jar they # point to, go under a directory named after the jar. if is_manifest(p): m = self.files[p] if self.files.contains(p) \ else ManifestFile(base) for e in parse_manifest(self.base, p, f.open()): m.add(self._handle_manifest_entry(e, jars)) if self.files.contains(p): continue f = m # If the file is a packed addon, unpack it under a directory named # after the xpi. if p.endswith('.xpi') and self._maybe_zip(f): self._fill_with_jar(p[:-4], self._open_jar(p, f)) continue if not p in jars: self.files.add(p, f)
def add_manifest(self, entry): # Store manifest entries in a single manifest per directory, named # after their parent directory, except for root manifests, all named # chrome.manifest. if entry.base: name = mozpath.basename(entry.base) else: name = 'chrome' path = mozpath.normpath(mozpath.join(entry.base, '%s.manifest' % name)) if not self.copier.contains(path): # Add a reference to the manifest file in the parent manifest, if # the manifest file is not a root manifest. if entry.base: parent = mozpath.dirname(entry.base) relbase = mozpath.basename(entry.base) relpath = mozpath.join(relbase, mozpath.basename(path)) self.add_manifest(Manifest(parent, relpath)) self.copier.add(path, ManifestFile(entry.base)) self.copier[path].add(entry)
def add_manifest(self, entry): # Store manifest entries in a single manifest per directory, named # after their parent directory, except for root manifests, all named # chrome.manifest. base = self._get_base(entry.base) if entry.base == base: name = 'chrome' else: name = mozpack.path.basename(entry.base) path = mozpack.path.normpath( mozpack.path.join(entry.base, '%s.manifest' % name)) if not self.copier.contains(path): assert mozpack.path.basedir(entry.base, [base]) == base # Add a reference to the manifest file in the parent manifest, if # the manifest file is not a root manifest. if len(entry.base) > len(base): parent = mozpack.path.dirname(entry.base) relbase = mozpack.path.basename(entry.base) relpath = mozpack.path.join(relbase, mozpack.path.basename(path)) FlatFormatter.add_manifest(self, Manifest(parent, relpath)) self.copier.add(path, ManifestFile(entry.base)) self.copier[path].add(entry)
def test_manifest_file(self): f = ManifestFile("chrome") f.add(ManifestContent("chrome", "global", "toolkit/content/global/")) f.add(ManifestResource("chrome", "gre-resources", "toolkit/res/")) f.add(ManifestResource("chrome/pdfjs", "pdfjs", "./")) f.add(ManifestContent("chrome/pdfjs", "pdfjs", "pdfjs")) f.add( ManifestLocale("chrome", "browser", "en-US", "en-US/locale/browser/")) f.copy(self.tmppath("chrome.manifest")) self.assertEqual( open(self.tmppath("chrome.manifest")).readlines(), [ "content global toolkit/content/global/\n", "resource gre-resources toolkit/res/\n", "resource pdfjs pdfjs/\n", "content pdfjs pdfjs/pdfjs\n", "locale browser en-US en-US/locale/browser/\n", ], ) self.assertRaises( ValueError, f.remove, ManifestContent("", "global", "toolkit/content/global/"), ) self.assertRaises( ValueError, f.remove, ManifestOverride( "chrome", "chrome://global/locale/netError.dtd", "chrome://browser/locale/netError.dtd", ), ) f.remove(ManifestContent("chrome", "global", "toolkit/content/global/")) self.assertRaises( ValueError, f.remove, ManifestContent("chrome", "global", "toolkit/content/global/"), ) f.copy(self.tmppath("chrome.manifest")) content = open(self.tmppath("chrome.manifest"), "rb").read() self.assertEqual(content[:42], f.open().read(42)) self.assertEqual(content, f.open().read())
def test_manifest_file(self): f = ManifestFile('chrome') f.add(ManifestContent('chrome', 'global', 'toolkit/content/global/')) f.add(ManifestResource('chrome', 'gre-resources', 'toolkit/res/')) f.add(ManifestResource('chrome/pdfjs', 'pdfjs', './')) f.add(ManifestContent('chrome/pdfjs', 'pdfjs', 'pdfjs')) f.add( ManifestLocale('chrome', 'browser', 'en-US', 'en-US/locale/browser/')) f.copy(self.tmppath('chrome.manifest')) self.assertEqual( open(self.tmppath('chrome.manifest')).readlines(), [ 'content global toolkit/content/global/\n', 'resource gre-resources toolkit/res/\n', 'resource pdfjs pdfjs/\n', 'content pdfjs pdfjs/pdfjs\n', 'locale browser en-US en-US/locale/browser/\n', ]) self.assertRaises( ValueError, f.remove, ManifestContent('', 'global', 'toolkit/content/global/')) self.assertRaises( ValueError, f.remove, ManifestOverride('chrome', 'chrome://global/locale/netError.dtd', 'chrome://browser/locale/netError.dtd')) f.remove(ManifestContent('chrome', 'global', 'toolkit/content/global/')) self.assertRaises( ValueError, f.remove, ManifestContent('chrome', 'global', 'toolkit/content/global/')) f.copy(self.tmppath('chrome.manifest')) content = open(self.tmppath('chrome.manifest')).read() self.assertEqual(content[:42], f.open().read(42)) self.assertEqual(content, f.open().read())
def repack(source, l10n): finder = UnpackFinder(source) l10n_finder = UnpackFinder(l10n) copier = FileCopier() if finder.kind == 'flat': formatter = FlatFormatter(copier) elif finder.kind == 'jar': formatter = JarFormatter(copier, optimize=finder.optimizedjars) elif finder.kind == 'omni': formatter = OmniJarFormatter(copier, finder.omnijar, optimize=finder.optimizedjars) # Read all manifest entries from the packaged directory. manifests = dict((p, m) for p, m in finder.find('**/*.manifest') if is_manifest(p)) assert all(isinstance(m, ManifestFile) for m in manifests.itervalues()) entries = [e for m in manifests.itervalues() for e in m if e.localized] # Find unique locales used in these manifest entries. locales = list(set(e.id for e in entries if isinstance(e, ManifestLocale))) # Find all paths whose manifest are included by no other manifest. includes = set(mozpack.path.join(e.base, e.relpath) for m in manifests.itervalues() for e in m if isinstance(e, Manifest)) bases = [mozpack.path.dirname(p) for p in set(manifests.keys()) - includes] # Read all manifest entries from the langpack directory. manifests = [m for p, m in l10n_finder.find('**/*.manifest') if is_manifest(p)] assert all(isinstance(m, ManifestFile) for m in manifests) l10n_entries = [e for m in manifests for e in m if e.localized] # Find unique locales used in these manifest entries. l10n_locales = list(set(e.id for e in l10n_entries if isinstance(e, ManifestLocale))) # The code further below assumes there's only one locale replaced with # another one. if len(locales) > 1 or len(l10n_locales) > 1: errors.fatal("Multiple locales aren't supported") locale = locales[0] l10n_locale = l10n_locales[0] # For each base directory, store what path a locale chrome package name # corresponds to. # e.g., for the following entry under app/chrome: # locale foo en-US path/to/files # keep track that the locale path for foo in app is # app/chrome/path/to/files. l10n_paths = {} for e in l10n_entries: if isinstance(e, ManifestChrome): base = mozpack.path.basedir(e.path, bases) if not base in l10n_paths: l10n_paths[base] = {} l10n_paths[base][e.name] = e.path # For chrome and non chrome files or directories, store what langpack path # corresponds to a package path. paths = dict((e.path, l10n_paths[mozpack.path.basedir(e.path, bases)][e.name]) for e in entries if isinstance(e, ManifestEntryWithRelPath)) for path in NON_CHROME: for p, f in l10n_finder.find(path): paths[p] = p # Create a new package, with non localized bits coming from the original # package, and localized bits coming from the langpack. packager = SimplePackager(formatter) for p, f in finder: if is_manifest(p): # Remove localized manifest entries. for e in [e for e in f if e.localized]: f.remove(e) base = mozpack.path.basedir(p, paths.keys()) if base: # If the path is one that needs a locale replacement, use the # corresponding file from the langpack. subpath = mozpack.path.relpath(p, base) path = mozpack.path.normpath(mozpack.path.join(paths[base], subpath)) files = [f for p, f in l10n_finder.find(path)] if len(files) == 0 and base in NON_CHROME: path = path.replace(locale, l10n_locale) files = [f for p, f in l10n_finder.find(path)] if len(files) == 0: if not base in NON_CHROME: errors.error("Missing file: %s" % os.path.join(l10n, path)) else: packager.add(path, files[0]) else: packager.add(p, f) # Add localized manifest entries from the langpack. l10n_manifests = [] for base in set(e.base for e in l10n_entries): m = ManifestFile(base) for e in l10n_entries: if e.base == base: m.add(e) path = mozpack.path.join(base, 'chrome.%s.manifest' % l10n_locale) l10n_manifests.append((path, m)) bases = packager.get_bases() for path, m in l10n_manifests: base = mozpack.path.basedir(path, bases) packager.add(path, m) # Add a "manifest $path" entry in the top manifest under that base. m = ManifestFile(base) m.add(Manifest(base, mozpack.path.relpath(path, base))) packager.add(mozpack.path.join(base, 'chrome.manifest'), m) packager.close() # Add any remaining non chrome files. for base in NON_CHROME: for p, f in l10n_finder.find(base): if not formatter.contains(p): formatter.add(p, f) # Transplant jar preloading information. for path, log in finder.jarlogs.iteritems(): assert isinstance(copier[path], Jarrer) copier[path].preload([l.replace(locale, l10n_locale) for l in log]) copier.copy(source) generate_precomplete(source)
def test_manifest_file(self): f = ManifestFile("chrome") f.add(ManifestContent("chrome", "global", "toolkit/content/global/")) f.add(ManifestResource("chrome", "gre-resources", "toolkit/res/")) f.add(ManifestResource("chrome/pdfjs", "pdfjs", "./")) f.add(ManifestContent("chrome/pdfjs", "pdfjs", "pdfjs")) f.add(ManifestLocale("chrome", "browser", "en-US", "en-US/locale/browser/")) f.copy(self.tmppath("chrome.manifest")) self.assertEqual( open(self.tmppath("chrome.manifest")).readlines(), [ "content global toolkit/content/global/\n", "resource gre-resources toolkit/res/\n", "resource pdfjs pdfjs/\n", "content pdfjs pdfjs/pdfjs\n", "locale browser en-US en-US/locale/browser/\n", ], ) self.assertRaises(ValueError, f.remove, ManifestContent("", "global", "toolkit/content/global/")) self.assertRaises( ValueError, f.remove, ManifestOverride("chrome", "chrome://global/locale/netError.dtd", "chrome://browser/locale/netError.dtd"), ) f.remove(ManifestContent("chrome", "global", "toolkit/content/global/")) self.assertRaises(ValueError, f.remove, ManifestContent("chrome", "global", "toolkit/content/global/")) f.copy(self.tmppath("chrome.manifest")) content = open(self.tmppath("chrome.manifest")).read() self.assertEqual(content[:42], f.open().read(42)) self.assertEqual(content, f.open().read())
def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()): app = LocaleManifestFinder(app_finder) l10n = LocaleManifestFinder(l10n_finder) # The code further below assumes there's only one locale replaced with # another one. if len(app.locales) > 1: errors.fatal("Multiple app locales aren't supported: " + ",".join(app.locales)) if len(l10n.locales) > 1: errors.fatal("Multiple l10n locales aren't supported: " + ",".join(l10n.locales)) locale = app.locales[0] l10n_locale = l10n.locales[0] # For each base directory, store what path a locale chrome package name # corresponds to. # e.g., for the following entry under app/chrome: # locale foo en-US path/to/files # keep track that the locale path for foo in app is # app/chrome/path/to/files. # As there may be multiple locale entries with the same base, but with # different flags, that tracking takes the flags into account when there # are some. Example: # locale foo en-US path/to/files/win os=Win # locale foo en-US path/to/files/mac os=Darwin def key(entry): if entry.flags: return '%s %s' % (entry.name, entry.flags) return entry.name l10n_paths = {} for e in l10n.entries: if isinstance(e, ManifestChrome): base = mozpath.basedir(e.path, app.bases) l10n_paths.setdefault(base, {}) l10n_paths[base][key(e)] = e.path # For chrome and non chrome files or directories, store what langpack path # corresponds to a package path. paths = {} for e in app.entries: if isinstance(e, ManifestEntryWithRelPath): base = mozpath.basedir(e.path, app.bases) if base not in l10n_paths: errors.fatal("Locale doesn't contain %s/" % base) # Allow errors to accumulate continue if key(e) not in l10n_paths[base]: errors.fatal("Locale doesn't have a manifest entry for '%s'" % e.name) # Allow errors to accumulate continue paths[e.path] = l10n_paths[base][key(e)] for pattern in non_chrome: for base in app.bases: path = mozpath.join(base, pattern) left = set(p for p, f in app_finder.find(path)) right = set(p for p, f in l10n_finder.find(path)) for p in right: paths[p] = p for p in left - right: paths[p] = None # Create a new package, with non localized bits coming from the original # package, and localized bits coming from the langpack. packager = SimplePackager(formatter) for p, f in app_finder: if is_manifest(p): # Remove localized manifest entries. for e in [e for e in f if e.localized]: f.remove(e) # If the path is one that needs a locale replacement, use the # corresponding file from the langpack. path = None if p in paths: path = paths[p] if not path: continue else: base = mozpath.basedir(p, paths.keys()) if base: subpath = mozpath.relpath(p, base) path = mozpath.normpath(mozpath.join(paths[base], subpath)) if path: files = [f for p, f in l10n_finder.find(path)] if not len(files): if base not in non_chrome: finderBase = "" if hasattr(l10n_finder, 'base'): finderBase = l10n_finder.base errors.error("Missing file: %s" % os.path.join(finderBase, path)) else: packager.add(path, files[0]) else: packager.add(p, f) # Add localized manifest entries from the langpack. l10n_manifests = [] for base in set(e.base for e in l10n.entries): m = ManifestFile(base, [e for e in l10n.entries if e.base == base]) path = mozpath.join(base, 'chrome.%s.manifest' % l10n_locale) l10n_manifests.append((path, m)) bases = packager.get_bases() for path, m in l10n_manifests: base = mozpath.basedir(path, bases) packager.add(path, m) # Add a "manifest $path" entry in the top manifest under that base. m = ManifestFile(base) m.add(Manifest(base, mozpath.relpath(path, base))) packager.add(mozpath.join(base, 'chrome.manifest'), m) packager.close() # Add any remaining non chrome files. for pattern in non_chrome: for base in bases: for p, f in l10n_finder.find(mozpath.join(base, pattern)): if not formatter.contains(p): formatter.add(p, f) # Resources in `localization` directories are packaged from the source and then # if localized versions are present in the l10n dir, we package them as well # keeping the source dir resources as a runtime fallback. for p, f in l10n_finder.find('**/localization'): if not formatter.contains(p): formatter.add(p, f) # Transplant jar preloading information. for path, log in six.iteritems(app_finder.jarlogs): assert isinstance(copier[path], Jarrer) copier[path].preload([l.replace(locale, l10n_locale) for l in log])
def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()): app = LocaleManifestFinder(app_finder) l10n = LocaleManifestFinder(l10n_finder) # The code further below assumes there's only one locale replaced with # another one. if len(app.locales) > 1 or len(l10n.locales) > 1: errors.fatal("Multiple locales aren't supported") locale = app.locales[0] l10n_locale = l10n.locales[0] # For each base directory, store what path a locale chrome package name # corresponds to. # e.g., for the following entry under app/chrome: # locale foo en-US path/to/files # keep track that the locale path for foo in app is # app/chrome/path/to/files. l10n_paths = {} for e in l10n.entries: if isinstance(e, ManifestChrome): base = mozpath.basedir(e.path, app.bases) l10n_paths.setdefault(base, {}) l10n_paths[base][e.name] = e.path # For chrome and non chrome files or directories, store what langpack path # corresponds to a package path. paths = {} for e in app.entries: if isinstance(e, ManifestEntryWithRelPath): base = mozpath.basedir(e.path, app.bases) if base not in l10n_paths: errors.fatal("Locale doesn't contain %s/" % base) # Allow errors to accumulate continue if e.name not in l10n_paths[base]: errors.fatal("Locale doesn't have a manifest entry for '%s'" % e.name) # Allow errors to accumulate continue paths[e.path] = l10n_paths[base][e.name] for pattern in non_chrome: for base in app.bases: path = mozpath.join(base, pattern) left = set(p for p, f in app_finder.find(path)) right = set(p for p, f in l10n_finder.find(path)) for p in right: paths[p] = p for p in left - right: paths[p] = None # Create a new package, with non localized bits coming from the original # package, and localized bits coming from the langpack. packager = SimplePackager(formatter) for p, f in app_finder: if is_manifest(p): # Remove localized manifest entries. for e in [e for e in f if e.localized]: f.remove(e) # If the path is one that needs a locale replacement, use the # corresponding file from the langpack. path = None if p in paths: path = paths[p] if not path: continue else: base = mozpath.basedir(p, paths.keys()) if base: subpath = mozpath.relpath(p, base) path = mozpath.normpath(mozpath.join(paths[base], subpath)) if path: files = [f for p, f in l10n_finder.find(path)] if not len(files): if base not in non_chrome: errors.error("Missing file: %s" % os.path.join(l10n_finder.base, path)) else: packager.add(path, files[0]) else: packager.add(p, f) # Add localized manifest entries from the langpack. l10n_manifests = [] for base in set(e.base for e in l10n.entries): m = ManifestFile(base, [e for e in l10n.entries if e.base == base]) path = mozpath.join(base, 'chrome.%s.manifest' % l10n_locale) l10n_manifests.append((path, m)) bases = packager.get_bases() for path, m in l10n_manifests: base = mozpath.basedir(path, bases) packager.add(path, m) # Add a "manifest $path" entry in the top manifest under that base. m = ManifestFile(base) m.add(Manifest(base, mozpath.relpath(path, base))) packager.add(mozpath.join(base, 'chrome.manifest'), m) packager.close() # Add any remaining non chrome files. for pattern in non_chrome: for base in bases: for p, f in l10n_finder.find(mozpath.join(base, pattern)): if not formatter.contains(p): formatter.add(p, f) # Transplant jar preloading information. for path, log in app_finder.jarlogs.iteritems(): assert isinstance(copier[path], Jarrer) copier[path].preload([l.replace(locale, l10n_locale) for l in log])
def test_l10n_repack(self): foo = GeneratedFile('foo') foobar = GeneratedFile('foobar') qux = GeneratedFile('qux') bar = GeneratedFile('bar') baz = GeneratedFile('baz') dict_aa = GeneratedFile('dict_aa') dict_bb = GeneratedFile('dict_bb') dict_cc = GeneratedFile('dict_cc') barbaz = GeneratedFile('barbaz') lst = GeneratedFile('foo\nbar') app_finder = MockFinder({ 'bar/foo': foo, 'chrome/foo/foobar': foobar, 'chrome/qux/qux.properties': qux, 'chrome/qux/baz/baz.properties': baz, 'chrome/chrome.manifest': ManifestFile('chrome', [ ManifestContent('chrome', 'foo', 'foo/'), ManifestLocale('chrome', 'qux', 'en-US', 'qux/'), ]), 'chrome.manifest': ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]), 'dict/aa': dict_aa, 'app/chrome/bar/barbaz.dtd': barbaz, 'app/chrome/chrome.manifest': ManifestFile( 'app/chrome', [ManifestLocale('app/chrome', 'bar', 'en-US', 'bar/')]), 'app/chrome.manifest': ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]), 'app/dict/bb': dict_bb, 'app/dict/cc': dict_cc, 'app/chrome/bar/search/foo.xml': foo, 'app/chrome/bar/search/bar.xml': bar, 'app/chrome/bar/search/lst.txt': lst, }) app_finder.jarlogs = {} app_finder.base = 'app' foo_l10n = GeneratedFile('foo_l10n') qux_l10n = GeneratedFile('qux_l10n') baz_l10n = GeneratedFile('baz_l10n') barbaz_l10n = GeneratedFile('barbaz_l10n') lst_l10n = GeneratedFile('foo\nqux') l10n_finder = MockFinder({ 'chrome/qux-l10n/qux.properties': qux_l10n, 'chrome/qux-l10n/baz/baz.properties': baz_l10n, 'chrome/chrome.manifest': ManifestFile('chrome', [ ManifestLocale('chrome', 'qux', 'x-test', 'qux-l10n/'), ]), 'chrome.manifest': ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]), 'dict/bb': dict_bb, 'dict/cc': dict_cc, 'app/chrome/bar-l10n/barbaz.dtd': barbaz_l10n, 'app/chrome/chrome.manifest': ManifestFile( 'app/chrome', [ManifestLocale('app/chrome', 'bar', 'x-test', 'bar-l10n/')]), 'app/chrome.manifest': ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]), 'app/dict/aa': dict_aa, 'app/chrome/bar-l10n/search/foo.xml': foo_l10n, 'app/chrome/bar-l10n/search/qux.xml': qux_l10n, 'app/chrome/bar-l10n/search/lst.txt': lst_l10n, }) l10n_finder.base = 'l10n' copier = FileRegistry() formatter = FlatFormatter(copier) l10n._repack(app_finder, l10n_finder, copier, formatter, ['dict', 'chrome/**/search/*.xml']) self.maxDiff = None repacked = { 'bar/foo': foo, 'chrome/foo/foobar': foobar, 'chrome/qux-l10n/qux.properties': qux_l10n, 'chrome/qux-l10n/baz/baz.properties': baz_l10n, 'chrome/chrome.manifest': ManifestFile('chrome', [ ManifestContent('chrome', 'foo', 'foo/'), ManifestLocale('chrome', 'qux', 'x-test', 'qux-l10n/'), ]), 'chrome.manifest': ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]), 'dict/bb': dict_bb, 'dict/cc': dict_cc, 'app/chrome/bar-l10n/barbaz.dtd': barbaz_l10n, 'app/chrome/chrome.manifest': ManifestFile( 'app/chrome', [ManifestLocale('app/chrome', 'bar', 'x-test', 'bar-l10n/')]), 'app/chrome.manifest': ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]), 'app/dict/aa': dict_aa, 'app/chrome/bar-l10n/search/foo.xml': foo_l10n, 'app/chrome/bar-l10n/search/qux.xml': qux_l10n, 'app/chrome/bar-l10n/search/lst.txt': lst_l10n, } self.assertEqual( dict((p, f.open().read()) for p, f in copier), dict((p, f.open().read()) for p, f in repacked.iteritems()))
def test_manifest_file(self): f = ManifestFile('chrome') f.add(ManifestContent('chrome', 'global', 'toolkit/content/global/')) f.add(ManifestResource('chrome', 'gre-resources', 'toolkit/res/')) f.add(ManifestResource('chrome/pdfjs', 'pdfjs', './')) f.add(ManifestContent('chrome/pdfjs', 'pdfjs', 'pdfjs')) f.add(ManifestLocale('chrome', 'browser', 'en-US', 'en-US/locale/browser/')) f.copy(self.tmppath('chrome.manifest')) self.assertEqual(open(self.tmppath('chrome.manifest')).readlines(), [ 'content global toolkit/content/global/\n', 'resource gre-resources toolkit/res/\n', 'resource pdfjs pdfjs/\n', 'content pdfjs pdfjs/pdfjs\n', 'locale browser en-US en-US/locale/browser/\n', ]) self.assertRaises( ValueError, f.remove, ManifestContent('', 'global', 'toolkit/content/global/') ) self.assertRaises( ValueError, f.remove, ManifestOverride('chrome', 'chrome://global/locale/netError.dtd', 'chrome://browser/locale/netError.dtd') ) f.remove(ManifestContent('chrome', 'global', 'toolkit/content/global/')) self.assertRaises( ValueError, f.remove, ManifestContent('chrome', 'global', 'toolkit/content/global/') ) f.copy(self.tmppath('chrome.manifest')) content = open(self.tmppath('chrome.manifest')).read() self.assertEqual(content[:42], f.open().read(42)) self.assertEqual(content, f.open().read())
def test_l10n_repack(self): foo = GeneratedFile(b"foo") foobar = GeneratedFile(b"foobar") qux = GeneratedFile(b"qux") bar = GeneratedFile(b"bar") baz = GeneratedFile(b"baz") dict_aa = GeneratedFile(b"dict_aa") dict_bb = GeneratedFile(b"dict_bb") dict_cc = GeneratedFile(b"dict_cc") barbaz = GeneratedFile(b"barbaz") lst = GeneratedFile(b"foo\nbar") app_finder = MockFinder({ "bar/foo": foo, "chrome/foo/foobar": foobar, "chrome/qux/qux.properties": qux, "chrome/qux/baz/baz.properties": baz, "chrome/chrome.manifest": ManifestFile( "chrome", [ ManifestContent("chrome", "foo", "foo/"), ManifestLocale("chrome", "qux", "en-US", "qux/"), ], ), "chrome.manifest": ManifestFile("", [Manifest("", "chrome/chrome.manifest")]), "dict/aa": dict_aa, "app/chrome/bar/barbaz.dtd": barbaz, "app/chrome/chrome.manifest": ManifestFile( "app/chrome", [ManifestLocale("app/chrome", "bar", "en-US", "bar/")]), "app/chrome.manifest": ManifestFile("app", [Manifest("app", "chrome/chrome.manifest")]), "app/dict/bb": dict_bb, "app/dict/cc": dict_cc, "app/chrome/bar/search/foo.xml": foo, "app/chrome/bar/search/bar.xml": bar, "app/chrome/bar/search/lst.txt": lst, }) app_finder.jarlogs = {} app_finder.base = "app" foo_l10n = GeneratedFile(b"foo_l10n") qux_l10n = GeneratedFile(b"qux_l10n") baz_l10n = GeneratedFile(b"baz_l10n") barbaz_l10n = GeneratedFile(b"barbaz_l10n") lst_l10n = GeneratedFile(b"foo\nqux") l10n_finder = MockFinder({ "chrome/qux-l10n/qux.properties": qux_l10n, "chrome/qux-l10n/baz/baz.properties": baz_l10n, "chrome/chrome.manifest": ManifestFile( "chrome", [ ManifestLocale("chrome", "qux", "x-test", "qux-l10n/"), ], ), "chrome.manifest": ManifestFile("", [Manifest("", "chrome/chrome.manifest")]), "dict/bb": dict_bb, "dict/cc": dict_cc, "app/chrome/bar-l10n/barbaz.dtd": barbaz_l10n, "app/chrome/chrome.manifest": ManifestFile( "app/chrome", [ManifestLocale("app/chrome", "bar", "x-test", "bar-l10n/")], ), "app/chrome.manifest": ManifestFile("app", [Manifest("app", "chrome/chrome.manifest")]), "app/dict/aa": dict_aa, "app/chrome/bar-l10n/search/foo.xml": foo_l10n, "app/chrome/bar-l10n/search/qux.xml": qux_l10n, "app/chrome/bar-l10n/search/lst.txt": lst_l10n, }) l10n_finder.base = "l10n" copier = FileRegistry() formatter = FlatFormatter(copier) l10n._repack( app_finder, l10n_finder, copier, formatter, ["dict", "chrome/**/search/*.xml"], ) self.maxDiff = None repacked = { "bar/foo": foo, "chrome/foo/foobar": foobar, "chrome/qux-l10n/qux.properties": qux_l10n, "chrome/qux-l10n/baz/baz.properties": baz_l10n, "chrome/chrome.manifest": ManifestFile( "chrome", [ ManifestContent("chrome", "foo", "foo/"), ManifestLocale("chrome", "qux", "x-test", "qux-l10n/"), ], ), "chrome.manifest": ManifestFile("", [Manifest("", "chrome/chrome.manifest")]), "dict/bb": dict_bb, "dict/cc": dict_cc, "app/chrome/bar-l10n/barbaz.dtd": barbaz_l10n, "app/chrome/chrome.manifest": ManifestFile( "app/chrome", [ManifestLocale("app/chrome", "bar", "x-test", "bar-l10n/")], ), "app/chrome.manifest": ManifestFile("app", [Manifest("app", "chrome/chrome.manifest")]), "app/dict/aa": dict_aa, "app/chrome/bar-l10n/search/foo.xml": foo_l10n, "app/chrome/bar-l10n/search/qux.xml": qux_l10n, "app/chrome/bar-l10n/search/lst.txt": lst_l10n, } self.assertEqual( dict((p, f.open().read()) for p, f in copier), dict((p, f.open().read()) for p, f in six.iteritems(repacked)), )
def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()): app = LocaleManifestFinder(app_finder) l10n = LocaleManifestFinder(l10n_finder) # The code further below assumes there's only one locale replaced with # another one. if len(app.locales) > 1 or len(l10n.locales) > 1: errors.fatal("Multiple locales aren't supported") locale = app.locales[0] l10n_locale = l10n.locales[0] # For each base directory, store what path a locale chrome package name # corresponds to. # e.g., for the following entry under app/chrome: # locale foo en-US path/to/files # keep track that the locale path for foo in app is # app/chrome/path/to/files. l10n_paths = {} for e in l10n.entries: if isinstance(e, ManifestChrome): base = mozpack.path.basedir(e.path, app.bases) l10n_paths.setdefault(base, {}) l10n_paths[base][e.name] = e.path # For chrome and non chrome files or directories, store what langpack path # corresponds to a package path. paths = dict((e.path, l10n_paths[mozpack.path.basedir(e.path, app.bases)][e.name]) for e in app.entries if isinstance(e, ManifestEntryWithRelPath)) for pattern in non_chrome: for base in app.bases: path = mozpack.path.join(base, pattern) left = set(p for p, f in app_finder.find(path)) right = set(p for p, f in l10n_finder.find(path)) for p in right: paths[p] = p for p in left - right: paths[p] = None # Create a new package, with non localized bits coming from the original # package, and localized bits coming from the langpack. packager = SimplePackager(formatter) for p, f in app_finder: if is_manifest(p): # Remove localized manifest entries. for e in [e for e in f if e.localized]: f.remove(e) # If the path is one that needs a locale replacement, use the # corresponding file from the langpack. path = None if p in paths: path = paths[p] if not path: continue else: base = mozpack.path.basedir(p, paths.keys()) if base: subpath = mozpack.path.relpath(p, base) path = mozpack.path.normpath(mozpack.path.join(paths[base], subpath)) if path: files = [f for p, f in l10n_finder.find(path)] if not len(files): if base not in non_chrome: errors.error("Missing file: %s" % os.path.join(l10n_finder.base, path)) else: packager.add(path, files[0]) else: packager.add(p, f) # Add localized manifest entries from the langpack. l10n_manifests = [] for base in set(e.base for e in l10n.entries): m = ManifestFile(base, [e for e in l10n.entries if e.base == base]) path = mozpack.path.join(base, 'chrome.%s.manifest' % l10n_locale) l10n_manifests.append((path, m)) bases = packager.get_bases() for path, m in l10n_manifests: base = mozpack.path.basedir(path, bases) packager.add(path, m) # Add a "manifest $path" entry in the top manifest under that base. m = ManifestFile(base) m.add(Manifest(base, mozpack.path.relpath(path, base))) packager.add(mozpack.path.join(base, 'chrome.manifest'), m) packager.close() # Add any remaining non chrome files. for pattern in non_chrome: for base in bases: for p, f in l10n_finder.find(mozpack.path.join(base, pattern)): if not formatter.contains(p): formatter.add(p, f) # Transplant jar preloading information. for path, log in app_finder.jarlogs.iteritems(): assert isinstance(copier[path], Jarrer) copier[path].preload([l.replace(locale, l10n_locale) for l in log])