def _add_base(self, base, addon=False):
if addon:
# Because add_base is always called with parents before children,
# all the possible ancestry of `base` is already present in
# `_sub_formatter`.
parent_base = mozpath.basedir(base, self._sub_formatter.keys())
rel_base = mozpath.relpath(base, parent_base)
# If the addon is under a resource directory, package it in the
# omnijar.
parent_sub_formatter = self._sub_formatter[parent_base]
if parent_sub_formatter.is_resource(rel_base):
omnijar_sub_formatter = \
parent_sub_formatter._sub_formatter[self._omnijar_name]
self._sub_formatter[base] = FlatSubFormatter(
FileRegistrySubtree(rel_base,
omnijar_sub_formatter.copier))
return
JarFormatter._add_base(self, base, addon)
else:
# Initialize a chrome.manifest next to the omnijar file so that
# there's always a chrome.manifest file, even an empty one.
path = mozpath.normpath(mozpath.join(base, 'chrome.manifest'))
if not self.copier.contains(path):
self.copier.add(path, ManifestFile(''))
self._sub_formatter[base] = OmniJarSubFormatter(
FileRegistrySubtree(base, self.copier), self._omnijar_name,
self._compress, self._non_resources)
def add_manifest(self, entry):
# Store manifest entries in a single manifest per directory, named
# after their parent directory, except for root manifests, all named
# chrome.manifest.
if entry.base:
name = mozpath.basename(entry.base)
else:
name = 'chrome'
path = mozpath.normpath(mozpath.join(entry.base, '%s.manifest' % name))
if not self.copier.contains(path):
# Add a reference to the manifest file in the parent manifest, if
# the manifest file is not a root manifest.
if entry.base:
parent = mozpath.dirname(entry.base)
relbase = mozpath.basename(entry.base)
relpath = mozpath.join(relbase,
mozpath.basename(path))
self.add_manifest(Manifest(parent, relpath))
self.copier.add(path, ManifestFile(entry.base))
if isinstance(entry, ManifestChrome):
data = self._chrome_db.setdefault(entry.name, {})
entries = data.setdefault(entry.type, [])
for e in entries:
# Ideally, we'd actually check whether entry.flags are more
# specific than e.flags, but in practice the following test
# is enough for now.
if not entry.flags or e.flags and entry.flags == e.flags:
errors.fatal('"%s" overrides "%s"' % (entry, e))
entries.append(entry)
self.copier[path].add(entry)
def __init__(self, source, omnijar_name=None, unpack_xpi=True):
if isinstance(source, BaseFinder):
self._finder = source
else:
self._finder = FileFinder(source)
self.base = self._finder.base
self.files = FileRegistry()
self.kind = "flat"
if omnijar_name:
self.omnijar = omnijar_name
else:
# Can't include globally because of bootstrapping issues.
from buildconfig import substs
self.omnijar = substs.get("OMNIJAR_NAME", "omni.ja")
self.jarlogs = {}
self.compressed = False
self._unpack_xpi = unpack_xpi
jars = set()
for p, f in self._finder.find("*"):
# Skip the precomplete file, which is generated at packaging time.
if p == "precomplete":
continue
base = mozpath.dirname(p)
# If the file matches the omnijar pattern, it is an omnijar.
# All the files it contains go in the directory containing the full
# pattern. Manifests are merged if there is a corresponding manifest
# in the directory.
if self._maybe_zip(f) and mozpath.match(p, "**/%s" % self.omnijar):
jar = self._open_jar(p, f)
if "chrome.manifest" in jar:
self.kind = "omni"
self._fill_with_jar(p[: -len(self.omnijar) - 1], jar)
continue
# If the file is a manifest, scan its entries for some referencing
# jar: urls. If there are some, the files contained in the jar they
# point to, go under a directory named after the jar.
if is_manifest(p):
m = self.files[p] if self.files.contains(p) else ManifestFile(base)
for e in parse_manifest(
self.base, p, codecs.getreader("utf-8")(f.open())
):
m.add(self._handle_manifest_entry(e, jars))
if self.files.contains(p):
continue
f = m
# If we're unpacking packed addons and the file is a packed addon,
# unpack it under a directory named after the xpi.
if self._unpack_xpi and p.endswith(".xpi") and self._maybe_zip(f):
self._fill_with_jar(p[:-4], self._open_jar(p, f))
continue
if p not in jars:
self.files.add(p, f)
def find(self, pattern):
for p, f in self._finder.find(pattern):
if isinstance(f, ManifestFile):
unwanted = [
e for e in f._entries
if isinstance(e, Manifest) and e.flags
]
if unwanted:
f = ManifestFile(
f._base,
[e for e in f._entries if e not in unwanted])
yield p, f
def _add_base(self, base, addon=False):
if addon:
JarFormatter._add_base(self, base, addon)
else:
# Initialize a chrome.manifest next to the omnijar file so that
# there's always a chrome.manifest file, even an empty one.
path = mozpath.normpath(mozpath.join(base, 'chrome.manifest'))
if not self.copier.contains(path):
self.copier.add(path, ManifestFile(''))
self._sub_formatter[base] = OmniJarSubFormatter(
FileRegistrySubtree(base, self.copier), self._omnijar_name,
self._compress, self._optimize, self._non_resources)
def _fill_with_omnijar(self, base, jar):
for j in jar:
path = mozpack.path.join(base, j.filename)
if is_manifest(j.filename):
m = self.files[path] if self.files.contains(path) \
else ManifestFile(mozpack.path.dirname(path))
for e in parse_manifest(None, path, j):
m.add(e)
if not self.files.contains(path):
self.files.add(path, m)
continue
else:
self.files.add(path, DeflatedFile(j))
def __init__(self, source):
if isinstance(source, BaseFinder):
self._finder = source
else:
self._finder = FileFinder(source)
self.base = self._finder.base
self.files = FileRegistry()
self.kind = 'flat'
self.omnijar = None
self.jarlogs = {}
self.optimizedjars = False
self.compressed = False
jars = set()
for p, f in self._finder.find('*'):
# Skip the precomplete file, which is generated at packaging time.
if p == 'precomplete':
continue
base = mozpath.dirname(p)
# If the file is a zip/jar that is not a .xpi, and contains a
# chrome.manifest, it is an omnijar. All the files it contains
# go in the directory containing the omnijar. Manifests are merged
# if there is a corresponding manifest in the directory.
if not p.endswith('.xpi') and self._maybe_zip(f) and \
(mozpath.basename(p) == self.omnijar or
not self.omnijar):
jar = self._open_jar(p, f)
if 'chrome.manifest' in jar:
self.kind = 'omni'
self.omnijar = mozpath.basename(p)
self._fill_with_jar(base, jar)
continue
# If the file is a manifest, scan its entries for some referencing
# jar: urls. If there are some, the files contained in the jar they
# point to, go under a directory named after the jar.
if is_manifest(p):
m = self.files[p] if self.files.contains(p) \
else ManifestFile(base)
for e in parse_manifest(self.base, p, f.open()):
m.add(self._handle_manifest_entry(e, jars))
if self.files.contains(p):
continue
f = m
# If the file is a packed addon, unpack it under a directory named
# after the xpi.
if p.endswith('.xpi') and self._maybe_zip(f):
self._fill_with_jar(p[:-4], self._open_jar(p, f))
continue
if not p in jars:
self.files.add(p, f)
def add_manifest(self, entry):
# Store manifest entries in a single manifest per directory, named
# after their parent directory, except for root manifests, all named
# chrome.manifest.
if entry.base:
name = mozpath.basename(entry.base)
else:
name = 'chrome'
path = mozpath.normpath(mozpath.join(entry.base, '%s.manifest' % name))
if not self.copier.contains(path):
# Add a reference to the manifest file in the parent manifest, if
# the manifest file is not a root manifest.
if entry.base:
parent = mozpath.dirname(entry.base)
relbase = mozpath.basename(entry.base)
relpath = mozpath.join(relbase, mozpath.basename(path))
self.add_manifest(Manifest(parent, relpath))
self.copier.add(path, ManifestFile(entry.base))
self.copier[path].add(entry)
def add_manifest(self, entry):
# Store manifest entries in a single manifest per directory, named
# after their parent directory, except for root manifests, all named
# chrome.manifest.
base = self._get_base(entry.base)
if entry.base == base:
name = 'chrome'
else:
name = mozpack.path.basename(entry.base)
path = mozpack.path.normpath(
mozpack.path.join(entry.base, '%s.manifest' % name))
if not self.copier.contains(path):
assert mozpack.path.basedir(entry.base, [base]) == base
# Add a reference to the manifest file in the parent manifest, if
# the manifest file is not a root manifest.
if len(entry.base) > len(base):
parent = mozpack.path.dirname(entry.base)
relbase = mozpack.path.basename(entry.base)
relpath = mozpack.path.join(relbase,
mozpack.path.basename(path))
FlatFormatter.add_manifest(self, Manifest(parent, relpath))
self.copier.add(path, ManifestFile(entry.base))
self.copier[path].add(entry)
def test_manifest_file(self):
f = ManifestFile("chrome")
f.add(ManifestContent("chrome", "global", "toolkit/content/global/"))
f.add(ManifestResource("chrome", "gre-resources", "toolkit/res/"))
f.add(ManifestResource("chrome/pdfjs", "pdfjs", "./"))
f.add(ManifestContent("chrome/pdfjs", "pdfjs", "pdfjs"))
f.add(
ManifestLocale("chrome", "browser", "en-US",
"en-US/locale/browser/"))
f.copy(self.tmppath("chrome.manifest"))
self.assertEqual(
open(self.tmppath("chrome.manifest")).readlines(),
[
"content global toolkit/content/global/\n",
"resource gre-resources toolkit/res/\n",
"resource pdfjs pdfjs/\n",
"content pdfjs pdfjs/pdfjs\n",
"locale browser en-US en-US/locale/browser/\n",
],
)
self.assertRaises(
ValueError,
f.remove,
ManifestContent("", "global", "toolkit/content/global/"),
)
self.assertRaises(
ValueError,
f.remove,
ManifestOverride(
"chrome",
"chrome://global/locale/netError.dtd",
"chrome://browser/locale/netError.dtd",
),
)
f.remove(ManifestContent("chrome", "global",
"toolkit/content/global/"))
self.assertRaises(
ValueError,
f.remove,
ManifestContent("chrome", "global", "toolkit/content/global/"),
)
f.copy(self.tmppath("chrome.manifest"))
content = open(self.tmppath("chrome.manifest"), "rb").read()
self.assertEqual(content[:42], f.open().read(42))
self.assertEqual(content, f.open().read())
def test_manifest_file(self):
f = ManifestFile('chrome')
f.add(ManifestContent('chrome', 'global', 'toolkit/content/global/'))
f.add(ManifestResource('chrome', 'gre-resources', 'toolkit/res/'))
f.add(ManifestResource('chrome/pdfjs', 'pdfjs', './'))
f.add(ManifestContent('chrome/pdfjs', 'pdfjs', 'pdfjs'))
f.add(
ManifestLocale('chrome', 'browser', 'en-US',
'en-US/locale/browser/'))
f.copy(self.tmppath('chrome.manifest'))
self.assertEqual(
open(self.tmppath('chrome.manifest')).readlines(), [
'content global toolkit/content/global/\n',
'resource gre-resources toolkit/res/\n',
'resource pdfjs pdfjs/\n',
'content pdfjs pdfjs/pdfjs\n',
'locale browser en-US en-US/locale/browser/\n',
])
self.assertRaises(
ValueError, f.remove,
ManifestContent('', 'global', 'toolkit/content/global/'))
self.assertRaises(
ValueError, f.remove,
ManifestOverride('chrome', 'chrome://global/locale/netError.dtd',
'chrome://browser/locale/netError.dtd'))
f.remove(ManifestContent('chrome', 'global',
'toolkit/content/global/'))
self.assertRaises(
ValueError, f.remove,
ManifestContent('chrome', 'global', 'toolkit/content/global/'))
f.copy(self.tmppath('chrome.manifest'))
content = open(self.tmppath('chrome.manifest')).read()
self.assertEqual(content[:42], f.open().read(42))
self.assertEqual(content, f.open().read())
def repack(source, l10n):
finder = UnpackFinder(source)
l10n_finder = UnpackFinder(l10n)
copier = FileCopier()
if finder.kind == 'flat':
formatter = FlatFormatter(copier)
elif finder.kind == 'jar':
formatter = JarFormatter(copier, optimize=finder.optimizedjars)
elif finder.kind == 'omni':
formatter = OmniJarFormatter(copier, finder.omnijar,
optimize=finder.optimizedjars)
# Read all manifest entries from the packaged directory.
manifests = dict((p, m) for p, m in finder.find('**/*.manifest')
if is_manifest(p))
assert all(isinstance(m, ManifestFile) for m in manifests.itervalues())
entries = [e for m in manifests.itervalues() for e in m if e.localized]
# Find unique locales used in these manifest entries.
locales = list(set(e.id for e in entries if isinstance(e, ManifestLocale)))
# Find all paths whose manifest are included by no other manifest.
includes = set(mozpack.path.join(e.base, e.relpath)
for m in manifests.itervalues()
for e in m if isinstance(e, Manifest))
bases = [mozpack.path.dirname(p) for p in set(manifests.keys()) - includes]
# Read all manifest entries from the langpack directory.
manifests = [m for p, m in l10n_finder.find('**/*.manifest')
if is_manifest(p)]
assert all(isinstance(m, ManifestFile) for m in manifests)
l10n_entries = [e for m in manifests for e in m if e.localized]
# Find unique locales used in these manifest entries.
l10n_locales = list(set(e.id for e in l10n_entries
if isinstance(e, ManifestLocale)))
# The code further below assumes there's only one locale replaced with
# another one.
if len(locales) > 1 or len(l10n_locales) > 1:
errors.fatal("Multiple locales aren't supported")
locale = locales[0]
l10n_locale = l10n_locales[0]
# For each base directory, store what path a locale chrome package name
# corresponds to.
# e.g., for the following entry under app/chrome:
# locale foo en-US path/to/files
# keep track that the locale path for foo in app is
# app/chrome/path/to/files.
l10n_paths = {}
for e in l10n_entries:
if isinstance(e, ManifestChrome):
base = mozpack.path.basedir(e.path, bases)
if not base in l10n_paths:
l10n_paths[base] = {}
l10n_paths[base][e.name] = e.path
# For chrome and non chrome files or directories, store what langpack path
# corresponds to a package path.
paths = dict((e.path,
l10n_paths[mozpack.path.basedir(e.path, bases)][e.name])
for e in entries if isinstance(e, ManifestEntryWithRelPath))
for path in NON_CHROME:
for p, f in l10n_finder.find(path):
paths[p] = p
# Create a new package, with non localized bits coming from the original
# package, and localized bits coming from the langpack.
packager = SimplePackager(formatter)
for p, f in finder:
if is_manifest(p):
# Remove localized manifest entries.
for e in [e for e in f if e.localized]:
f.remove(e)
base = mozpack.path.basedir(p, paths.keys())
if base:
# If the path is one that needs a locale replacement, use the
# corresponding file from the langpack.
subpath = mozpack.path.relpath(p, base)
path = mozpack.path.normpath(mozpack.path.join(paths[base],
subpath))
files = [f for p, f in l10n_finder.find(path)]
if len(files) == 0 and base in NON_CHROME:
path = path.replace(locale, l10n_locale)
files = [f for p, f in l10n_finder.find(path)]
if len(files) == 0:
if not base in NON_CHROME:
errors.error("Missing file: %s" % os.path.join(l10n, path))
else:
packager.add(path, files[0])
else:
packager.add(p, f)
# Add localized manifest entries from the langpack.
l10n_manifests = []
for base in set(e.base for e in l10n_entries):
m = ManifestFile(base)
for e in l10n_entries:
if e.base == base:
m.add(e)
path = mozpack.path.join(base, 'chrome.%s.manifest' % l10n_locale)
l10n_manifests.append((path, m))
bases = packager.get_bases()
for path, m in l10n_manifests:
base = mozpack.path.basedir(path, bases)
packager.add(path, m)
# Add a "manifest $path" entry in the top manifest under that base.
m = ManifestFile(base)
m.add(Manifest(base, mozpack.path.relpath(path, base)))
packager.add(mozpack.path.join(base, 'chrome.manifest'), m)
packager.close()
# Add any remaining non chrome files.
for base in NON_CHROME:
for p, f in l10n_finder.find(base):
if not formatter.contains(p):
formatter.add(p, f)
# Transplant jar preloading information.
for path, log in finder.jarlogs.iteritems():
assert isinstance(copier[path], Jarrer)
copier[path].preload([l.replace(locale, l10n_locale) for l in log])
copier.copy(source)
generate_precomplete(source)
def test_manifest_file(self):
f = ManifestFile("chrome")
f.add(ManifestContent("chrome", "global", "toolkit/content/global/"))
f.add(ManifestResource("chrome", "gre-resources", "toolkit/res/"))
f.add(ManifestResource("chrome/pdfjs", "pdfjs", "./"))
f.add(ManifestContent("chrome/pdfjs", "pdfjs", "pdfjs"))
f.add(ManifestLocale("chrome", "browser", "en-US", "en-US/locale/browser/"))
f.copy(self.tmppath("chrome.manifest"))
self.assertEqual(
open(self.tmppath("chrome.manifest")).readlines(),
[
"content global toolkit/content/global/\n",
"resource gre-resources toolkit/res/\n",
"resource pdfjs pdfjs/\n",
"content pdfjs pdfjs/pdfjs\n",
"locale browser en-US en-US/locale/browser/\n",
],
)
self.assertRaises(ValueError, f.remove, ManifestContent("", "global", "toolkit/content/global/"))
self.assertRaises(
ValueError,
f.remove,
ManifestOverride("chrome", "chrome://global/locale/netError.dtd", "chrome://browser/locale/netError.dtd"),
)
f.remove(ManifestContent("chrome", "global", "toolkit/content/global/"))
self.assertRaises(ValueError, f.remove, ManifestContent("chrome", "global", "toolkit/content/global/"))
f.copy(self.tmppath("chrome.manifest"))
content = open(self.tmppath("chrome.manifest")).read()
self.assertEqual(content[:42], f.open().read(42))
self.assertEqual(content, f.open().read())
def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()):
app = LocaleManifestFinder(app_finder)
l10n = LocaleManifestFinder(l10n_finder)
# The code further below assumes there's only one locale replaced with
# another one.
if len(app.locales) > 1:
errors.fatal("Multiple app locales aren't supported: " +
",".join(app.locales))
if len(l10n.locales) > 1:
errors.fatal("Multiple l10n locales aren't supported: " +
",".join(l10n.locales))
locale = app.locales[0]
l10n_locale = l10n.locales[0]
# For each base directory, store what path a locale chrome package name
# corresponds to.
# e.g., for the following entry under app/chrome:
# locale foo en-US path/to/files
# keep track that the locale path for foo in app is
# app/chrome/path/to/files.
# As there may be multiple locale entries with the same base, but with
# different flags, that tracking takes the flags into account when there
# are some. Example:
# locale foo en-US path/to/files/win os=Win
# locale foo en-US path/to/files/mac os=Darwin
def key(entry):
if entry.flags:
return '%s %s' % (entry.name, entry.flags)
return entry.name
l10n_paths = {}
for e in l10n.entries:
if isinstance(e, ManifestChrome):
base = mozpath.basedir(e.path, app.bases)
l10n_paths.setdefault(base, {})
l10n_paths[base][key(e)] = e.path
# For chrome and non chrome files or directories, store what langpack path
# corresponds to a package path.
paths = {}
for e in app.entries:
if isinstance(e, ManifestEntryWithRelPath):
base = mozpath.basedir(e.path, app.bases)
if base not in l10n_paths:
errors.fatal("Locale doesn't contain %s/" % base)
# Allow errors to accumulate
continue
if key(e) not in l10n_paths[base]:
errors.fatal("Locale doesn't have a manifest entry for '%s'" %
e.name)
# Allow errors to accumulate
continue
paths[e.path] = l10n_paths[base][key(e)]
for pattern in non_chrome:
for base in app.bases:
path = mozpath.join(base, pattern)
left = set(p for p, f in app_finder.find(path))
right = set(p for p, f in l10n_finder.find(path))
for p in right:
paths[p] = p
for p in left - right:
paths[p] = None
# Create a new package, with non localized bits coming from the original
# package, and localized bits coming from the langpack.
packager = SimplePackager(formatter)
for p, f in app_finder:
if is_manifest(p):
# Remove localized manifest entries.
for e in [e for e in f if e.localized]:
f.remove(e)
# If the path is one that needs a locale replacement, use the
# corresponding file from the langpack.
path = None
if p in paths:
path = paths[p]
if not path:
continue
else:
base = mozpath.basedir(p, paths.keys())
if base:
subpath = mozpath.relpath(p, base)
path = mozpath.normpath(mozpath.join(paths[base], subpath))
if path:
files = [f for p, f in l10n_finder.find(path)]
if not len(files):
if base not in non_chrome:
finderBase = ""
if hasattr(l10n_finder, 'base'):
finderBase = l10n_finder.base
errors.error("Missing file: %s" %
os.path.join(finderBase, path))
else:
packager.add(path, files[0])
else:
packager.add(p, f)
# Add localized manifest entries from the langpack.
l10n_manifests = []
for base in set(e.base for e in l10n.entries):
m = ManifestFile(base, [e for e in l10n.entries if e.base == base])
path = mozpath.join(base, 'chrome.%s.manifest' % l10n_locale)
l10n_manifests.append((path, m))
bases = packager.get_bases()
for path, m in l10n_manifests:
base = mozpath.basedir(path, bases)
packager.add(path, m)
# Add a "manifest $path" entry in the top manifest under that base.
m = ManifestFile(base)
m.add(Manifest(base, mozpath.relpath(path, base)))
packager.add(mozpath.join(base, 'chrome.manifest'), m)
packager.close()
# Add any remaining non chrome files.
for pattern in non_chrome:
for base in bases:
for p, f in l10n_finder.find(mozpath.join(base, pattern)):
if not formatter.contains(p):
formatter.add(p, f)
# Resources in `localization` directories are packaged from the source and then
# if localized versions are present in the l10n dir, we package them as well
# keeping the source dir resources as a runtime fallback.
for p, f in l10n_finder.find('**/localization'):
if not formatter.contains(p):
formatter.add(p, f)
# Transplant jar preloading information.
for path, log in six.iteritems(app_finder.jarlogs):
assert isinstance(copier[path], Jarrer)
copier[path].preload([l.replace(locale, l10n_locale) for l in log])
def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()):
app = LocaleManifestFinder(app_finder)
l10n = LocaleManifestFinder(l10n_finder)
# The code further below assumes there's only one locale replaced with
# another one.
if len(app.locales) > 1 or len(l10n.locales) > 1:
errors.fatal("Multiple locales aren't supported")
locale = app.locales[0]
l10n_locale = l10n.locales[0]
# For each base directory, store what path a locale chrome package name
# corresponds to.
# e.g., for the following entry under app/chrome:
# locale foo en-US path/to/files
# keep track that the locale path for foo in app is
# app/chrome/path/to/files.
l10n_paths = {}
for e in l10n.entries:
if isinstance(e, ManifestChrome):
base = mozpath.basedir(e.path, app.bases)
l10n_paths.setdefault(base, {})
l10n_paths[base][e.name] = e.path
# For chrome and non chrome files or directories, store what langpack path
# corresponds to a package path.
paths = {}
for e in app.entries:
if isinstance(e, ManifestEntryWithRelPath):
base = mozpath.basedir(e.path, app.bases)
if base not in l10n_paths:
errors.fatal("Locale doesn't contain %s/" % base)
# Allow errors to accumulate
continue
if e.name not in l10n_paths[base]:
errors.fatal("Locale doesn't have a manifest entry for '%s'" %
e.name)
# Allow errors to accumulate
continue
paths[e.path] = l10n_paths[base][e.name]
for pattern in non_chrome:
for base in app.bases:
path = mozpath.join(base, pattern)
left = set(p for p, f in app_finder.find(path))
right = set(p for p, f in l10n_finder.find(path))
for p in right:
paths[p] = p
for p in left - right:
paths[p] = None
# Create a new package, with non localized bits coming from the original
# package, and localized bits coming from the langpack.
packager = SimplePackager(formatter)
for p, f in app_finder:
if is_manifest(p):
# Remove localized manifest entries.
for e in [e for e in f if e.localized]:
f.remove(e)
# If the path is one that needs a locale replacement, use the
# corresponding file from the langpack.
path = None
if p in paths:
path = paths[p]
if not path:
continue
else:
base = mozpath.basedir(p, paths.keys())
if base:
subpath = mozpath.relpath(p, base)
path = mozpath.normpath(mozpath.join(paths[base],
subpath))
if path:
files = [f for p, f in l10n_finder.find(path)]
if not len(files):
if base not in non_chrome:
errors.error("Missing file: %s" %
os.path.join(l10n_finder.base, path))
else:
packager.add(path, files[0])
else:
packager.add(p, f)
# Add localized manifest entries from the langpack.
l10n_manifests = []
for base in set(e.base for e in l10n.entries):
m = ManifestFile(base, [e for e in l10n.entries if e.base == base])
path = mozpath.join(base, 'chrome.%s.manifest' % l10n_locale)
l10n_manifests.append((path, m))
bases = packager.get_bases()
for path, m in l10n_manifests:
base = mozpath.basedir(path, bases)
packager.add(path, m)
# Add a "manifest $path" entry in the top manifest under that base.
m = ManifestFile(base)
m.add(Manifest(base, mozpath.relpath(path, base)))
packager.add(mozpath.join(base, 'chrome.manifest'), m)
packager.close()
# Add any remaining non chrome files.
for pattern in non_chrome:
for base in bases:
for p, f in l10n_finder.find(mozpath.join(base, pattern)):
if not formatter.contains(p):
formatter.add(p, f)
# Transplant jar preloading information.
for path, log in app_finder.jarlogs.iteritems():
assert isinstance(copier[path], Jarrer)
copier[path].preload([l.replace(locale, l10n_locale) for l in log])
def test_l10n_repack(self):
foo = GeneratedFile('foo')
foobar = GeneratedFile('foobar')
qux = GeneratedFile('qux')
bar = GeneratedFile('bar')
baz = GeneratedFile('baz')
dict_aa = GeneratedFile('dict_aa')
dict_bb = GeneratedFile('dict_bb')
dict_cc = GeneratedFile('dict_cc')
barbaz = GeneratedFile('barbaz')
lst = GeneratedFile('foo\nbar')
app_finder = MockFinder({
'bar/foo':
foo,
'chrome/foo/foobar':
foobar,
'chrome/qux/qux.properties':
qux,
'chrome/qux/baz/baz.properties':
baz,
'chrome/chrome.manifest':
ManifestFile('chrome', [
ManifestContent('chrome', 'foo', 'foo/'),
ManifestLocale('chrome', 'qux', 'en-US', 'qux/'),
]),
'chrome.manifest':
ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]),
'dict/aa':
dict_aa,
'app/chrome/bar/barbaz.dtd':
barbaz,
'app/chrome/chrome.manifest':
ManifestFile(
'app/chrome',
[ManifestLocale('app/chrome', 'bar', 'en-US', 'bar/')]),
'app/chrome.manifest':
ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]),
'app/dict/bb':
dict_bb,
'app/dict/cc':
dict_cc,
'app/chrome/bar/search/foo.xml':
foo,
'app/chrome/bar/search/bar.xml':
bar,
'app/chrome/bar/search/lst.txt':
lst,
})
app_finder.jarlogs = {}
app_finder.base = 'app'
foo_l10n = GeneratedFile('foo_l10n')
qux_l10n = GeneratedFile('qux_l10n')
baz_l10n = GeneratedFile('baz_l10n')
barbaz_l10n = GeneratedFile('barbaz_l10n')
lst_l10n = GeneratedFile('foo\nqux')
l10n_finder = MockFinder({
'chrome/qux-l10n/qux.properties':
qux_l10n,
'chrome/qux-l10n/baz/baz.properties':
baz_l10n,
'chrome/chrome.manifest':
ManifestFile('chrome', [
ManifestLocale('chrome', 'qux', 'x-test', 'qux-l10n/'),
]),
'chrome.manifest':
ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]),
'dict/bb':
dict_bb,
'dict/cc':
dict_cc,
'app/chrome/bar-l10n/barbaz.dtd':
barbaz_l10n,
'app/chrome/chrome.manifest':
ManifestFile(
'app/chrome',
[ManifestLocale('app/chrome', 'bar', 'x-test', 'bar-l10n/')]),
'app/chrome.manifest':
ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]),
'app/dict/aa':
dict_aa,
'app/chrome/bar-l10n/search/foo.xml':
foo_l10n,
'app/chrome/bar-l10n/search/qux.xml':
qux_l10n,
'app/chrome/bar-l10n/search/lst.txt':
lst_l10n,
})
l10n_finder.base = 'l10n'
copier = FileRegistry()
formatter = FlatFormatter(copier)
l10n._repack(app_finder, l10n_finder, copier, formatter,
['dict', 'chrome/**/search/*.xml'])
self.maxDiff = None
repacked = {
'bar/foo':
foo,
'chrome/foo/foobar':
foobar,
'chrome/qux-l10n/qux.properties':
qux_l10n,
'chrome/qux-l10n/baz/baz.properties':
baz_l10n,
'chrome/chrome.manifest':
ManifestFile('chrome', [
ManifestContent('chrome', 'foo', 'foo/'),
ManifestLocale('chrome', 'qux', 'x-test', 'qux-l10n/'),
]),
'chrome.manifest':
ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]),
'dict/bb':
dict_bb,
'dict/cc':
dict_cc,
'app/chrome/bar-l10n/barbaz.dtd':
barbaz_l10n,
'app/chrome/chrome.manifest':
ManifestFile(
'app/chrome',
[ManifestLocale('app/chrome', 'bar', 'x-test', 'bar-l10n/')]),
'app/chrome.manifest':
ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]),
'app/dict/aa':
dict_aa,
'app/chrome/bar-l10n/search/foo.xml':
foo_l10n,
'app/chrome/bar-l10n/search/qux.xml':
qux_l10n,
'app/chrome/bar-l10n/search/lst.txt':
lst_l10n,
}
self.assertEqual(
dict((p, f.open().read()) for p, f in copier),
dict((p, f.open().read()) for p, f in repacked.iteritems()))
def test_manifest_file(self):
f = ManifestFile('chrome')
f.add(ManifestContent('chrome', 'global', 'toolkit/content/global/'))
f.add(ManifestResource('chrome', 'gre-resources', 'toolkit/res/'))
f.add(ManifestResource('chrome/pdfjs', 'pdfjs', './'))
f.add(ManifestContent('chrome/pdfjs', 'pdfjs', 'pdfjs'))
f.add(ManifestLocale('chrome', 'browser', 'en-US',
'en-US/locale/browser/'))
f.copy(self.tmppath('chrome.manifest'))
self.assertEqual(open(self.tmppath('chrome.manifest')).readlines(), [
'content global toolkit/content/global/\n',
'resource gre-resources toolkit/res/\n',
'resource pdfjs pdfjs/\n',
'content pdfjs pdfjs/pdfjs\n',
'locale browser en-US en-US/locale/browser/\n',
])
self.assertRaises(
ValueError,
f.remove,
ManifestContent('', 'global', 'toolkit/content/global/')
)
self.assertRaises(
ValueError,
f.remove,
ManifestOverride('chrome', 'chrome://global/locale/netError.dtd',
'chrome://browser/locale/netError.dtd')
)
f.remove(ManifestContent('chrome', 'global',
'toolkit/content/global/'))
self.assertRaises(
ValueError,
f.remove,
ManifestContent('chrome', 'global', 'toolkit/content/global/')
)
f.copy(self.tmppath('chrome.manifest'))
content = open(self.tmppath('chrome.manifest')).read()
self.assertEqual(content[:42], f.open().read(42))
self.assertEqual(content, f.open().read())
def test_l10n_repack(self):
foo = GeneratedFile(b"foo")
foobar = GeneratedFile(b"foobar")
qux = GeneratedFile(b"qux")
bar = GeneratedFile(b"bar")
baz = GeneratedFile(b"baz")
dict_aa = GeneratedFile(b"dict_aa")
dict_bb = GeneratedFile(b"dict_bb")
dict_cc = GeneratedFile(b"dict_cc")
barbaz = GeneratedFile(b"barbaz")
lst = GeneratedFile(b"foo\nbar")
app_finder = MockFinder({
"bar/foo":
foo,
"chrome/foo/foobar":
foobar,
"chrome/qux/qux.properties":
qux,
"chrome/qux/baz/baz.properties":
baz,
"chrome/chrome.manifest":
ManifestFile(
"chrome",
[
ManifestContent("chrome", "foo", "foo/"),
ManifestLocale("chrome", "qux", "en-US", "qux/"),
],
),
"chrome.manifest":
ManifestFile("", [Manifest("", "chrome/chrome.manifest")]),
"dict/aa":
dict_aa,
"app/chrome/bar/barbaz.dtd":
barbaz,
"app/chrome/chrome.manifest":
ManifestFile(
"app/chrome",
[ManifestLocale("app/chrome", "bar", "en-US", "bar/")]),
"app/chrome.manifest":
ManifestFile("app", [Manifest("app", "chrome/chrome.manifest")]),
"app/dict/bb":
dict_bb,
"app/dict/cc":
dict_cc,
"app/chrome/bar/search/foo.xml":
foo,
"app/chrome/bar/search/bar.xml":
bar,
"app/chrome/bar/search/lst.txt":
lst,
})
app_finder.jarlogs = {}
app_finder.base = "app"
foo_l10n = GeneratedFile(b"foo_l10n")
qux_l10n = GeneratedFile(b"qux_l10n")
baz_l10n = GeneratedFile(b"baz_l10n")
barbaz_l10n = GeneratedFile(b"barbaz_l10n")
lst_l10n = GeneratedFile(b"foo\nqux")
l10n_finder = MockFinder({
"chrome/qux-l10n/qux.properties":
qux_l10n,
"chrome/qux-l10n/baz/baz.properties":
baz_l10n,
"chrome/chrome.manifest":
ManifestFile(
"chrome",
[
ManifestLocale("chrome", "qux", "x-test", "qux-l10n/"),
],
),
"chrome.manifest":
ManifestFile("", [Manifest("", "chrome/chrome.manifest")]),
"dict/bb":
dict_bb,
"dict/cc":
dict_cc,
"app/chrome/bar-l10n/barbaz.dtd":
barbaz_l10n,
"app/chrome/chrome.manifest":
ManifestFile(
"app/chrome",
[ManifestLocale("app/chrome", "bar", "x-test", "bar-l10n/")],
),
"app/chrome.manifest":
ManifestFile("app", [Manifest("app", "chrome/chrome.manifest")]),
"app/dict/aa":
dict_aa,
"app/chrome/bar-l10n/search/foo.xml":
foo_l10n,
"app/chrome/bar-l10n/search/qux.xml":
qux_l10n,
"app/chrome/bar-l10n/search/lst.txt":
lst_l10n,
})
l10n_finder.base = "l10n"
copier = FileRegistry()
formatter = FlatFormatter(copier)
l10n._repack(
app_finder,
l10n_finder,
copier,
formatter,
["dict", "chrome/**/search/*.xml"],
)
self.maxDiff = None
repacked = {
"bar/foo":
foo,
"chrome/foo/foobar":
foobar,
"chrome/qux-l10n/qux.properties":
qux_l10n,
"chrome/qux-l10n/baz/baz.properties":
baz_l10n,
"chrome/chrome.manifest":
ManifestFile(
"chrome",
[
ManifestContent("chrome", "foo", "foo/"),
ManifestLocale("chrome", "qux", "x-test", "qux-l10n/"),
],
),
"chrome.manifest":
ManifestFile("", [Manifest("", "chrome/chrome.manifest")]),
"dict/bb":
dict_bb,
"dict/cc":
dict_cc,
"app/chrome/bar-l10n/barbaz.dtd":
barbaz_l10n,
"app/chrome/chrome.manifest":
ManifestFile(
"app/chrome",
[ManifestLocale("app/chrome", "bar", "x-test", "bar-l10n/")],
),
"app/chrome.manifest":
ManifestFile("app", [Manifest("app", "chrome/chrome.manifest")]),
"app/dict/aa":
dict_aa,
"app/chrome/bar-l10n/search/foo.xml":
foo_l10n,
"app/chrome/bar-l10n/search/qux.xml":
qux_l10n,
"app/chrome/bar-l10n/search/lst.txt":
lst_l10n,
}
self.assertEqual(
dict((p, f.open().read()) for p, f in copier),
dict((p, f.open().read()) for p, f in six.iteritems(repacked)),
)
def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()):
app = LocaleManifestFinder(app_finder)
l10n = LocaleManifestFinder(l10n_finder)
# The code further below assumes there's only one locale replaced with
# another one.
if len(app.locales) > 1 or len(l10n.locales) > 1:
errors.fatal("Multiple locales aren't supported")
locale = app.locales[0]
l10n_locale = l10n.locales[0]
# For each base directory, store what path a locale chrome package name
# corresponds to.
# e.g., for the following entry under app/chrome:
# locale foo en-US path/to/files
# keep track that the locale path for foo in app is
# app/chrome/path/to/files.
l10n_paths = {}
for e in l10n.entries:
if isinstance(e, ManifestChrome):
base = mozpack.path.basedir(e.path, app.bases)
l10n_paths.setdefault(base, {})
l10n_paths[base][e.name] = e.path
# For chrome and non chrome files or directories, store what langpack path
# corresponds to a package path.
paths = dict((e.path,
l10n_paths[mozpack.path.basedir(e.path, app.bases)][e.name])
for e in app.entries
if isinstance(e, ManifestEntryWithRelPath))
for pattern in non_chrome:
for base in app.bases:
path = mozpack.path.join(base, pattern)
left = set(p for p, f in app_finder.find(path))
right = set(p for p, f in l10n_finder.find(path))
for p in right:
paths[p] = p
for p in left - right:
paths[p] = None
# Create a new package, with non localized bits coming from the original
# package, and localized bits coming from the langpack.
packager = SimplePackager(formatter)
for p, f in app_finder:
if is_manifest(p):
# Remove localized manifest entries.
for e in [e for e in f if e.localized]:
f.remove(e)
# If the path is one that needs a locale replacement, use the
# corresponding file from the langpack.
path = None
if p in paths:
path = paths[p]
if not path:
continue
else:
base = mozpack.path.basedir(p, paths.keys())
if base:
subpath = mozpack.path.relpath(p, base)
path = mozpack.path.normpath(mozpack.path.join(paths[base],
subpath))
if path:
files = [f for p, f in l10n_finder.find(path)]
if not len(files):
if base not in non_chrome:
errors.error("Missing file: %s" %
os.path.join(l10n_finder.base, path))
else:
packager.add(path, files[0])
else:
packager.add(p, f)
# Add localized manifest entries from the langpack.
l10n_manifests = []
for base in set(e.base for e in l10n.entries):
m = ManifestFile(base, [e for e in l10n.entries if e.base == base])
path = mozpack.path.join(base, 'chrome.%s.manifest' % l10n_locale)
l10n_manifests.append((path, m))
bases = packager.get_bases()
for path, m in l10n_manifests:
base = mozpack.path.basedir(path, bases)
packager.add(path, m)
# Add a "manifest $path" entry in the top manifest under that base.
m = ManifestFile(base)
m.add(Manifest(base, mozpack.path.relpath(path, base)))
packager.add(mozpack.path.join(base, 'chrome.manifest'), m)
packager.close()
# Add any remaining non chrome files.
for pattern in non_chrome:
for base in bases:
for p, f in l10n_finder.find(mozpack.path.join(base, pattern)):
if not formatter.contains(p):
formatter.add(p, f)
# Transplant jar preloading information.
for path, log in app_finder.jarlogs.iteritems():
assert isinstance(copier[path], Jarrer)
copier[path].preload([l.replace(locale, l10n_locale) for l in log])
