def unpack_to_registry(source, registry):
'''
Transform a jar chrome or omnijar packaged directory into a flat package.
The given registry is filled with the flat package.
'''
finder = UnpackFinder(source)
packager = SimplePackager(FlatFormatter(registry))
for p, f in finder.find('*'):
packager.add(p, f)
packager.close()
def unpack(source):
'''
Transform a jar chrome or omnijar packaged directory into a flat package.
'''
copier = FileCopier()
finder = UnpackFinder(source)
packager = SimplePackager(FlatFormatter(copier))
for p, f in finder.find('*'):
if mozpack.path.split(p)[0] not in STARTUP_CACHE_PATHS:
packager.add(p, f)
packager.close()
copier.copy(source, skip_if_older=False)
def unpack_to_registry(source, registry):
'''
Transform a jar chrome or omnijar packaged directory into a flat package.
The given registry is filled with the flat package.
'''
finder = UnpackFinder(source)
packager = SimplePackager(FlatFormatter(registry))
for p, f in finder.find('*'):
if mozpath.split(p)[0] not in STARTUP_CACHE_PATHS:
packager.add(p, f)
packager.close()
def unpack_to_registry(source, registry, omnijar_name=None):
"""
Transform a jar chrome or omnijar packaged directory into a flat package.
The given registry is filled with the flat package.
"""
finder = UnpackFinder(source, omnijar_name)
packager = SimplePackager(FlatFormatter(registry))
for p, f in finder.find("*"):
packager.add(p, f)
packager.close()
def unpack_to_registry(source, registry):
'''
Transform a jar chrome or omnijar packaged directory into a flat package.
The given registry is filled with the flat package.
'''
finder = UnpackFinder(source)
packager = SimplePackager(FlatFormatter(registry))
for p, f in finder.find('*'):
packager.add(p, f)
packager.close()
def unpack_to_registry(source, registry):
'''
Transform a jar chrome or omnijar packaged directory into a flat package.
The given registry is filled with the flat package.
'''
finder = UnpackFinder(source)
packager = SimplePackager(FlatFormatter(registry))
for p, f in finder.find('*'):
if mozpath.split(p)[0] not in STARTUP_CACHE_PATHS:
packager.add(p, f)
packager.close()
def unpack(source):
'''
Transform a jar chrome or omnijar packaged directory into a flat package.
'''
copier = FileCopier()
finder = UnpackFinder(source)
packager = SimplePackager(FlatFormatter(copier))
for p, f in finder.find('*'):
if mozpack.path.split(p)[0] not in STARTUP_CACHE_PATHS:
packager.add(p, f)
packager.close()
copier.copy(source, skip_if_older=False)
def main():
parser = argparse.ArgumentParser(
description=
"Merge two builds of a Gecko-based application into a Universal build")
parser.add_argument("app1", help="Directory containing the application")
parser.add_argument("app2",
help="Directory containing the application to merge")
parser.add_argument(
"--non-resource",
nargs="+",
metavar="PATTERN",
default=[],
help="Extra files not to be considered as resources",
)
options = parser.parse_args()
buildconfig.substs["OS_ARCH"] = "Darwin"
buildconfig.substs["LIPO"] = os.environ.get("LIPO")
app1_finder = UnpackFinder(FileFinder(options.app1, find_executables=True))
app2_finder = UnpackFinder(FileFinder(options.app2, find_executables=True))
app_finder = UnifiedBuildFinderWasmHack(app1_finder, app2_finder)
copier = FileCopier()
compress = min(app1_finder.compressed, JAR_DEFLATED)
if app1_finder.kind == "flat":
formatter = FlatFormatter(copier)
elif app1_finder.kind == "jar":
formatter = JarFormatter(copier, compress=compress)
elif app1_finder.kind == "omni":
formatter = OmniJarFormatter(
copier,
app1_finder.omnijar,
compress=compress,
non_resources=options.non_resource,
)
with errors.accumulate():
packager = SimplePackager(formatter)
for p, f in app_finder:
packager.add(p, f)
packager.close()
# Transplant jar preloading information.
for path, log in six.iteritems(app1_finder.jarlogs):
assert isinstance(copier[path], Jarrer)
copier[path].preload(log)
copier.copy(options.app1, skip_if_older=False)
def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()):
app = LocaleManifestFinder(app_finder)
l10n = LocaleManifestFinder(l10n_finder)
# The code further below assumes there's only one locale replaced with
# another one.
if len(app.locales) > 1:
errors.fatal("Multiple app locales aren't supported: " +
",".join(app.locales))
if len(l10n.locales) > 1:
errors.fatal("Multiple l10n locales aren't supported: " +
",".join(l10n.locales))
locale = app.locales[0]
l10n_locale = l10n.locales[0]
# For each base directory, store what path a locale chrome package name
# corresponds to.
# e.g., for the following entry under app/chrome:
# locale foo en-US path/to/files
# keep track that the locale path for foo in app is
# app/chrome/path/to/files.
# As there may be multiple locale entries with the same base, but with
# different flags, that tracking takes the flags into account when there
# are some. Example:
# locale foo en-US path/to/files/win os=Win
# locale foo en-US path/to/files/mac os=Darwin
def key(entry):
if entry.flags:
return '%s %s' % (entry.name, entry.flags)
return entry.name
l10n_paths = {}
for e in l10n.entries:
if isinstance(e, ManifestChrome):
base = mozpath.basedir(e.path, app.bases)
l10n_paths.setdefault(base, {})
l10n_paths[base][key(e)] = e.path
# For chrome and non chrome files or directories, store what langpack path
# corresponds to a package path.
paths = {}
for e in app.entries:
if isinstance(e, ManifestEntryWithRelPath):
base = mozpath.basedir(e.path, app.bases)
if base not in l10n_paths:
errors.fatal("Locale doesn't contain %s/" % base)
# Allow errors to accumulate
continue
if key(e) not in l10n_paths[base]:
errors.fatal("Locale doesn't have a manifest entry for '%s'" %
e.name)
# Allow errors to accumulate
continue
paths[e.path] = l10n_paths[base][key(e)]
for pattern in non_chrome:
for base in app.bases:
path = mozpath.join(base, pattern)
left = set(p for p, f in app_finder.find(path))
right = set(p for p, f in l10n_finder.find(path))
for p in right:
paths[p] = p
for p in left - right:
paths[p] = None
# Create a new package, with non localized bits coming from the original
# package, and localized bits coming from the langpack.
packager = SimplePackager(formatter)
for p, f in app_finder:
if is_manifest(p):
# Remove localized manifest entries.
for e in [e for e in f if e.localized]:
f.remove(e)
# If the path is one that needs a locale replacement, use the
# corresponding file from the langpack.
path = None
if p in paths:
path = paths[p]
if not path:
continue
else:
base = mozpath.basedir(p, paths.keys())
if base:
subpath = mozpath.relpath(p, base)
path = mozpath.normpath(mozpath.join(paths[base], subpath))
if path:
files = [f for p, f in l10n_finder.find(path)]
if not len(files):
if base not in non_chrome:
finderBase = ""
if hasattr(l10n_finder, 'base'):
finderBase = l10n_finder.base
errors.error("Missing file: %s" %
os.path.join(finderBase, path))
else:
packager.add(path, files[0])
else:
packager.add(p, f)
# Add localized manifest entries from the langpack.
l10n_manifests = []
for base in set(e.base for e in l10n.entries):
m = ManifestFile(base, [e for e in l10n.entries if e.base == base])
path = mozpath.join(base, 'chrome.%s.manifest' % l10n_locale)
l10n_manifests.append((path, m))
bases = packager.get_bases()
for path, m in l10n_manifests:
base = mozpath.basedir(path, bases)
packager.add(path, m)
# Add a "manifest $path" entry in the top manifest under that base.
m = ManifestFile(base)
m.add(Manifest(base, mozpath.relpath(path, base)))
packager.add(mozpath.join(base, 'chrome.manifest'), m)
packager.close()
# Add any remaining non chrome files.
for pattern in non_chrome:
for base in bases:
for p, f in l10n_finder.find(mozpath.join(base, pattern)):
if not formatter.contains(p):
formatter.add(p, f)
# Resources in `localization` directories are packaged from the source and then
# if localized versions are present in the l10n dir, we package them as well
# keeping the source dir resources as a runtime fallback.
for p, f in l10n_finder.find('**/localization'):
if not formatter.contains(p):
formatter.add(p, f)
# Transplant jar preloading information.
for path, log in six.iteritems(app_finder.jarlogs):
assert isinstance(copier[path], Jarrer)
copier[path].preload([l.replace(locale, l10n_locale) for l in log])
def test_simple_packager_manifest_consistency(self):
formatter = MockFormatter()
# bar/ is detected as an addon because of install.rdf, but top-level
# includes a manifest inside bar/.
packager = SimplePackager(formatter)
packager.add('base.manifest', GeneratedFile(
'manifest foo/bar.manifest\n'
'manifest bar/baz.manifest\n'
))
packager.add('foo/bar.manifest', GeneratedFile('resource bar bar'))
packager.add('bar/baz.manifest', GeneratedFile('resource baz baz'))
packager.add('bar/install.rdf', GeneratedFile(''))
with self.assertRaises(ErrorMessage) as e:
packager.close()
self.assertEqual(e.exception.message,
'Error: "bar/baz.manifest" is included from "base.manifest", '
'which is outside "bar"')
# bar/ is detected as a separate base because of chrome.manifest that
# is included nowhere, but top-level includes another manifest inside
# bar/.
packager = SimplePackager(formatter)
packager.add('base.manifest', GeneratedFile(
'manifest foo/bar.manifest\n'
'manifest bar/baz.manifest\n'
))
packager.add('foo/bar.manifest', GeneratedFile('resource bar bar'))
packager.add('bar/baz.manifest', GeneratedFile('resource baz baz'))
packager.add('bar/chrome.manifest', GeneratedFile('resource baz baz'))
with self.assertRaises(ErrorMessage) as e:
packager.close()
self.assertEqual(e.exception.message,
'Error: "bar/baz.manifest" is included from "base.manifest", '
'which is outside "bar"')
# bar/ is detected as a separate base because of chrome.manifest that
# is included nowhere, but chrome.manifest includes baz.manifest from
# the same directory. This shouldn't error out.
packager = SimplePackager(formatter)
packager.add('base.manifest', GeneratedFile(
'manifest foo/bar.manifest\n'
))
packager.add('foo/bar.manifest', GeneratedFile('resource bar bar'))
packager.add('bar/baz.manifest', GeneratedFile('resource baz baz'))
packager.add('bar/chrome.manifest',
GeneratedFile('manifest baz.manifest'))
packager.close()
def test_simple_packager(self):
class GeneratedFileWithPath(GeneratedFile):
def __init__(self, path, content):
GeneratedFile.__init__(self, content)
self.path = path
formatter = MockFormatter()
packager = SimplePackager(formatter)
curdir = os.path.abspath(os.curdir)
file = GeneratedFileWithPath(os.path.join(curdir, 'foo',
'bar.manifest'),
'resource bar bar/\ncontent bar bar/')
with errors.context('manifest', 1):
packager.add('foo/bar.manifest', file)
file = GeneratedFileWithPath(os.path.join(curdir, 'foo',
'baz.manifest'),
'resource baz baz/')
with errors.context('manifest', 2):
packager.add('bar/baz.manifest', file)
with errors.context('manifest', 3):
packager.add('qux/qux.manifest',
GeneratedFile('resource qux qux/'))
bar_xpt = GeneratedFile('bar.xpt')
qux_xpt = GeneratedFile('qux.xpt')
foo_html = GeneratedFile('foo_html')
bar_html = GeneratedFile('bar_html')
with errors.context('manifest', 4):
packager.add('foo/bar.xpt', bar_xpt)
with errors.context('manifest', 5):
packager.add('foo/bar/foo.html', foo_html)
packager.add('foo/bar/bar.html', bar_html)
file = GeneratedFileWithPath(os.path.join(curdir, 'foo.manifest'),
''.join([
'manifest foo/bar.manifest\n',
'manifest bar/baz.manifest\n',
]))
with errors.context('manifest', 6):
packager.add('foo.manifest', file)
with errors.context('manifest', 7):
packager.add('foo/qux.xpt', qux_xpt)
file = GeneratedFileWithPath(os.path.join(curdir, 'addon',
'chrome.manifest'),
'resource hoge hoge/')
with errors.context('manifest', 8):
packager.add('addon/chrome.manifest', file)
install_rdf = GeneratedFile('<RDF></RDF>')
with errors.context('manifest', 9):
packager.add('addon/install.rdf', install_rdf)
self.assertEqual(formatter.log, [])
with errors.context('dummy', 1):
packager.close()
self.maxDiff = None
# The formatter is expected to reorder the manifest entries so that
# chrome entries appear before the others.
self.assertEqual(formatter.log, [
(('dummy', 1), 'add_base', 'qux', False),
(('dummy', 1), 'add_base', 'addon', True),
((os.path.join(curdir, 'foo', 'bar.manifest'), 2),
'add_manifest', ManifestContent('foo', 'bar', 'bar/')),
((os.path.join(curdir, 'foo', 'bar.manifest'), 1),
'add_manifest', ManifestResource('foo', 'bar', 'bar/')),
(('bar/baz.manifest', 1),
'add_manifest', ManifestResource('bar', 'baz', 'baz/')),
(('qux/qux.manifest', 1),
'add_manifest', ManifestResource('qux', 'qux', 'qux/')),
(('manifest', 4), 'add_interfaces', 'foo/bar.xpt', bar_xpt),
(('manifest', 7), 'add_interfaces', 'foo/qux.xpt', qux_xpt),
((os.path.join(curdir, 'addon', 'chrome.manifest'), 1),
'add_manifest', ManifestResource('addon', 'hoge', 'hoge/')),
(('manifest', 5), 'add', 'foo/bar/foo.html', foo_html),
(('manifest', 5), 'add', 'foo/bar/bar.html', bar_html),
(('manifest', 9), 'add', 'addon/install.rdf', install_rdf),
])
self.assertEqual(packager.get_bases(), set(['', 'addon', 'qux']))
self.assertEqual(packager.get_bases(addons=False), set(['', 'qux']))
def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()):
app = LocaleManifestFinder(app_finder)
l10n = LocaleManifestFinder(l10n_finder)
# The code further below assumes there's only one locale replaced with
# another one.
if len(app.locales) > 1 or len(l10n.locales) > 1:
errors.fatal("Multiple locales aren't supported")
locale = app.locales[0]
l10n_locale = l10n.locales[0]
# For each base directory, store what path a locale chrome package name
# corresponds to.
# e.g., for the following entry under app/chrome:
# locale foo en-US path/to/files
# keep track that the locale path for foo in app is
# app/chrome/path/to/files.
l10n_paths = {}
for e in l10n.entries:
if isinstance(e, ManifestChrome):
base = mozpath.basedir(e.path, app.bases)
l10n_paths.setdefault(base, {})
l10n_paths[base][e.name] = e.path
# For chrome and non chrome files or directories, store what langpack path
# corresponds to a package path.
paths = {}
for e in app.entries:
if isinstance(e, ManifestEntryWithRelPath):
base = mozpath.basedir(e.path, app.bases)
if base not in l10n_paths:
errors.fatal("Locale doesn't contain %s/" % base)
# Allow errors to accumulate
continue
if e.name not in l10n_paths[base]:
errors.fatal("Locale doesn't have a manifest entry for '%s'" %
e.name)
# Allow errors to accumulate
continue
paths[e.path] = l10n_paths[base][e.name]
for pattern in non_chrome:
for base in app.bases:
path = mozpath.join(base, pattern)
left = set(p for p, f in app_finder.find(path))
right = set(p for p, f in l10n_finder.find(path))
for p in right:
paths[p] = p
for p in left - right:
paths[p] = None
# Create a new package, with non localized bits coming from the original
# package, and localized bits coming from the langpack.
packager = SimplePackager(formatter)
for p, f in app_finder:
if is_manifest(p):
# Remove localized manifest entries.
for e in [e for e in f if e.localized]:
f.remove(e)
# If the path is one that needs a locale replacement, use the
# corresponding file from the langpack.
path = None
if p in paths:
path = paths[p]
if not path:
continue
else:
base = mozpath.basedir(p, paths.keys())
if base:
subpath = mozpath.relpath(p, base)
path = mozpath.normpath(mozpath.join(paths[base],
subpath))
if path:
files = [f for p, f in l10n_finder.find(path)]
if not len(files):
if base not in non_chrome:
errors.error("Missing file: %s" %
os.path.join(l10n_finder.base, path))
else:
packager.add(path, files[0])
else:
packager.add(p, f)
# Add localized manifest entries from the langpack.
l10n_manifests = []
for base in set(e.base for e in l10n.entries):
m = ManifestFile(base, [e for e in l10n.entries if e.base == base])
path = mozpath.join(base, 'chrome.%s.manifest' % l10n_locale)
l10n_manifests.append((path, m))
bases = packager.get_bases()
for path, m in l10n_manifests:
base = mozpath.basedir(path, bases)
packager.add(path, m)
# Add a "manifest $path" entry in the top manifest under that base.
m = ManifestFile(base)
m.add(Manifest(base, mozpath.relpath(path, base)))
packager.add(mozpath.join(base, 'chrome.manifest'), m)
packager.close()
# Add any remaining non chrome files.
for pattern in non_chrome:
for base in bases:
for p, f in l10n_finder.find(mozpath.join(base, pattern)):
if not formatter.contains(p):
formatter.add(p, f)
# Transplant jar preloading information.
for path, log in app_finder.jarlogs.iteritems():
assert isinstance(copier[path], Jarrer)
copier[path].preload([l.replace(locale, l10n_locale) for l in log])
def test_simple_packager(self):
class GeneratedFileWithPath(GeneratedFile):
def __init__(self, path, content):
GeneratedFile.__init__(self, content)
self.path = path
formatter = MockFormatter()
packager = SimplePackager(formatter)
curdir = os.path.abspath(os.curdir)
file = GeneratedFileWithPath(os.path.join(curdir, 'foo',
'bar.manifest'),
'resource bar bar/\ncontent bar bar/')
with errors.context('manifest', 1):
packager.add('foo/bar.manifest', file)
file = GeneratedFileWithPath(os.path.join(curdir, 'foo',
'baz.manifest'),
'resource baz baz/')
with errors.context('manifest', 2):
packager.add('bar/baz.manifest', file)
with errors.context('manifest', 3):
packager.add('qux/qux.manifest',
GeneratedFile(''.join([
'resource qux qux/\n',
'binary-component qux.so\n',
])))
bar_xpt = GeneratedFile('bar.xpt')
qux_xpt = GeneratedFile('qux.xpt')
foo_html = GeneratedFile('foo_html')
bar_html = GeneratedFile('bar_html')
with errors.context('manifest', 4):
packager.add('foo/bar.xpt', bar_xpt)
with errors.context('manifest', 5):
packager.add('foo/bar/foo.html', foo_html)
packager.add('foo/bar/bar.html', bar_html)
file = GeneratedFileWithPath(os.path.join(curdir, 'foo.manifest'),
''.join([
'manifest foo/bar.manifest\n',
'manifest bar/baz.manifest\n',
]))
with errors.context('manifest', 6):
packager.add('foo.manifest', file)
with errors.context('manifest', 7):
packager.add('foo/qux.xpt', qux_xpt)
file = GeneratedFileWithPath(os.path.join(curdir, 'addon',
'chrome.manifest'),
'resource hoge hoge/')
with errors.context('manifest', 8):
packager.add('addon/chrome.manifest', file)
install_rdf = GeneratedFile('<RDF></RDF>')
with errors.context('manifest', 9):
packager.add('addon/install.rdf', install_rdf)
with errors.context('manifest', 10):
packager.add('addon2/install.rdf', install_rdf)
packager.add('addon2/chrome.manifest',
GeneratedFile('binary-component addon2.so'))
with errors.context('manifest', 11):
packager.add('addon3/install.rdf', install_rdf)
packager.add('addon3/chrome.manifest', GeneratedFile(
'manifest components/components.manifest'))
packager.add('addon3/components/components.manifest',
GeneratedFile('binary-component addon3.so'))
with errors.context('manifest', 12):
install_rdf_addon4 = GeneratedFile(
'<RDF>\n<...>\n<em:unpack>true</em:unpack>\n<...>\n</RDF>')
packager.add('addon4/install.rdf', install_rdf_addon4)
with errors.context('manifest', 13):
install_rdf_addon5 = GeneratedFile(
'<RDF>\n<...>\n<em:unpack>false</em:unpack>\n<...>\n</RDF>')
packager.add('addon5/install.rdf', install_rdf_addon5)
with errors.context('manifest', 14):
install_rdf_addon6 = GeneratedFile(
'<RDF>\n<... em:unpack=true>\n<...>\n</RDF>')
packager.add('addon6/install.rdf', install_rdf_addon6)
with errors.context('manifest', 15):
install_rdf_addon7 = GeneratedFile(
'<RDF>\n<... em:unpack=false>\n<...>\n</RDF>')
packager.add('addon7/install.rdf', install_rdf_addon7)
with errors.context('manifest', 16):
install_rdf_addon8 = GeneratedFile(
'<RDF>\n<... em:unpack="true">\n<...>\n</RDF>')
packager.add('addon8/install.rdf', install_rdf_addon8)
with errors.context('manifest', 17):
install_rdf_addon9 = GeneratedFile(
'<RDF>\n<... em:unpack="false">\n<...>\n</RDF>')
packager.add('addon9/install.rdf', install_rdf_addon9)
with errors.context('manifest', 18):
install_rdf_addon10 = GeneratedFile(
'<RDF>\n<... em:unpack=\'true\'>\n<...>\n</RDF>')
packager.add('addon10/install.rdf', install_rdf_addon10)
with errors.context('manifest', 19):
install_rdf_addon11 = GeneratedFile(
'<RDF>\n<... em:unpack=\'false\'>\n<...>\n</RDF>')
packager.add('addon11/install.rdf', install_rdf_addon11)
self.assertEqual(formatter.log, [])
with errors.context('dummy', 1):
packager.close()
self.maxDiff = None
# The formatter is expected to reorder the manifest entries so that
# chrome entries appear before the others.
self.assertEqual(formatter.log, [
(('dummy', 1), 'add_base', '', False),
(('dummy', 1), 'add_base', 'addon', True),
(('dummy', 1), 'add_base', 'addon10', 'unpacked'),
(('dummy', 1), 'add_base', 'addon11', True),
(('dummy', 1), 'add_base', 'addon2', 'unpacked'),
(('dummy', 1), 'add_base', 'addon3', 'unpacked'),
(('dummy', 1), 'add_base', 'addon4', 'unpacked'),
(('dummy', 1), 'add_base', 'addon5', True),
(('dummy', 1), 'add_base', 'addon6', 'unpacked'),
(('dummy', 1), 'add_base', 'addon7', True),
(('dummy', 1), 'add_base', 'addon8', 'unpacked'),
(('dummy', 1), 'add_base', 'addon9', True),
(('dummy', 1), 'add_base', 'qux', False),
((os.path.join(curdir, 'foo', 'bar.manifest'), 2),
'add_manifest', ManifestContent('foo', 'bar', 'bar/')),
((os.path.join(curdir, 'foo', 'bar.manifest'), 1),
'add_manifest', ManifestResource('foo', 'bar', 'bar/')),
(('bar/baz.manifest', 1),
'add_manifest', ManifestResource('bar', 'baz', 'baz/')),
(('qux/qux.manifest', 1),
'add_manifest', ManifestResource('qux', 'qux', 'qux/')),
(('qux/qux.manifest', 2),
'add_manifest', ManifestBinaryComponent('qux', 'qux.so')),
(('manifest', 4), 'add_interfaces', 'foo/bar.xpt', bar_xpt),
(('manifest', 7), 'add_interfaces', 'foo/qux.xpt', qux_xpt),
((os.path.join(curdir, 'addon', 'chrome.manifest'), 1),
'add_manifest', ManifestResource('addon', 'hoge', 'hoge/')),
(('addon2/chrome.manifest', 1), 'add_manifest',
ManifestBinaryComponent('addon2', 'addon2.so')),
(('addon3/components/components.manifest', 1), 'add_manifest',
ManifestBinaryComponent('addon3/components', 'addon3.so')),
(('manifest', 5), 'add', 'foo/bar/foo.html', foo_html),
(('manifest', 5), 'add', 'foo/bar/bar.html', bar_html),
(('manifest', 9), 'add', 'addon/install.rdf', install_rdf),
(('manifest', 10), 'add', 'addon2/install.rdf', install_rdf),
(('manifest', 11), 'add', 'addon3/install.rdf', install_rdf),
(('manifest', 12), 'add', 'addon4/install.rdf',
install_rdf_addon4),
(('manifest', 13), 'add', 'addon5/install.rdf',
install_rdf_addon5),
(('manifest', 14), 'add', 'addon6/install.rdf',
install_rdf_addon6),
(('manifest', 15), 'add', 'addon7/install.rdf',
install_rdf_addon7),
(('manifest', 16), 'add', 'addon8/install.rdf',
install_rdf_addon8),
(('manifest', 17), 'add', 'addon9/install.rdf',
install_rdf_addon9),
(('manifest', 18), 'add', 'addon10/install.rdf',
install_rdf_addon10),
(('manifest', 19), 'add', 'addon11/install.rdf',
install_rdf_addon11),
])
self.assertEqual(packager.get_bases(),
set(['', 'addon', 'addon2', 'addon3', 'addon4',
'addon5', 'addon6', 'addon7', 'addon8',
'addon9', 'addon10', 'addon11', 'qux']))
self.assertEqual(packager.get_bases(addons=False), set(['', 'qux']))
def test_simple_packager(self):
class GeneratedFileWithPath(GeneratedFile):
def __init__(self, path, content):
GeneratedFile.__init__(self, content)
self.path = path
formatter = MockFormatter()
packager = SimplePackager(formatter)
curdir = os.path.abspath(os.curdir)
file = GeneratedFileWithPath(os.path.join(curdir, 'foo',
'bar.manifest'),
'resource bar bar/\ncontent bar bar/')
with errors.context('manifest', 1):
packager.add('foo/bar.manifest', file)
file = GeneratedFileWithPath(os.path.join(curdir, 'foo',
'baz.manifest'),
'resource baz baz/')
with errors.context('manifest', 2):
packager.add('bar/baz.manifest', file)
with errors.context('manifest', 3):
packager.add('qux/qux.manifest',
GeneratedFile(''.join([
'resource qux qux/\n',
'binary-component qux.so\n',
])))
bar_xpt = GeneratedFile('bar.xpt')
qux_xpt = GeneratedFile('qux.xpt')
foo_html = GeneratedFile('foo_html')
bar_html = GeneratedFile('bar_html')
with errors.context('manifest', 4):
packager.add('foo/bar.xpt', bar_xpt)
with errors.context('manifest', 5):
packager.add('foo/bar/foo.html', foo_html)
packager.add('foo/bar/bar.html', bar_html)
file = GeneratedFileWithPath(os.path.join(curdir, 'foo.manifest'),
''.join([
'manifest foo/bar.manifest\n',
'manifest bar/baz.manifest\n',
]))
with errors.context('manifest', 6):
packager.add('foo.manifest', file)
with errors.context('manifest', 7):
packager.add('foo/qux.xpt', qux_xpt)
file = GeneratedFileWithPath(os.path.join(curdir, 'addon',
'chrome.manifest'),
'resource hoge hoge/')
with errors.context('manifest', 8):
packager.add('addon/chrome.manifest', file)
install_rdf = GeneratedFile('<RDF></RDF>')
with errors.context('manifest', 9):
packager.add('addon/install.rdf', install_rdf)
with errors.context('manifest', 10):
packager.add('addon2/install.rdf', install_rdf)
packager.add('addon2/chrome.manifest',
GeneratedFile('binary-component addon2.so'))
with errors.context('manifest', 11):
packager.add('addon3/install.rdf', install_rdf)
packager.add('addon3/chrome.manifest', GeneratedFile(
'manifest components/components.manifest'))
packager.add('addon3/components/components.manifest',
GeneratedFile('binary-component addon3.so'))
with errors.context('manifest', 12):
install_rdf_addon4 = GeneratedFile(
'<RDF>\n<...>\n<em:unpack>true</em:unpack>\n<...>\n</RDF>')
packager.add('addon4/install.rdf', install_rdf_addon4)
with errors.context('manifest', 13):
install_rdf_addon5 = GeneratedFile(
'<RDF>\n<...>\n<em:unpack>false</em:unpack>\n<...>\n</RDF>')
packager.add('addon5/install.rdf', install_rdf_addon5)
with errors.context('manifest', 14):
install_rdf_addon6 = GeneratedFile(
'<RDF>\n<... em:unpack=true>\n<...>\n</RDF>')
packager.add('addon6/install.rdf', install_rdf_addon6)
with errors.context('manifest', 15):
install_rdf_addon7 = GeneratedFile(
'<RDF>\n<... em:unpack=false>\n<...>\n</RDF>')
packager.add('addon7/install.rdf', install_rdf_addon7)
with errors.context('manifest', 16):
install_rdf_addon8 = GeneratedFile(
'<RDF>\n<... em:unpack="true">\n<...>\n</RDF>')
packager.add('addon8/install.rdf', install_rdf_addon8)
with errors.context('manifest', 17):
install_rdf_addon9 = GeneratedFile(
'<RDF>\n<... em:unpack="false">\n<...>\n</RDF>')
packager.add('addon9/install.rdf', install_rdf_addon9)
with errors.context('manifest', 18):
install_rdf_addon10 = GeneratedFile(
'<RDF>\n<... em:unpack=\'true\'>\n<...>\n</RDF>')
packager.add('addon10/install.rdf', install_rdf_addon10)
with errors.context('manifest', 19):
install_rdf_addon11 = GeneratedFile(
'<RDF>\n<... em:unpack=\'false\'>\n<...>\n</RDF>')
packager.add('addon11/install.rdf', install_rdf_addon11)
we_manifest = GeneratedFile(
'{"manifest_version": 2, "name": "Test WebExtension", "version": "1.0"}')
# hybrid and hybrid2 are both bootstrapped extensions with
# embedded webextensions, they differ in the order in which
# the manifests are added to the packager.
with errors.context('manifest', 20):
packager.add('hybrid/install.rdf', install_rdf)
with errors.context('manifest', 21):
packager.add('hybrid/webextension/manifest.json', we_manifest)
with errors.context('manifest', 22):
packager.add('hybrid2/webextension/manifest.json', we_manifest)
with errors.context('manifest', 23):
packager.add('hybrid2/install.rdf', install_rdf)
with errors.context('manifest', 24):
packager.add('webextension/manifest.json', we_manifest)
non_we_manifest = GeneratedFile('{"not a webextension": true}')
with errors.context('manifest', 25):
packager.add('nonwebextension/manifest.json', non_we_manifest)
self.assertEqual(formatter.log, [])
with errors.context('dummy', 1):
packager.close()
self.maxDiff = None
# The formatter is expected to reorder the manifest entries so that
# chrome entries appear before the others.
self.assertEqual(formatter.log, [
(('dummy', 1), 'add_base', '', False),
(('dummy', 1), 'add_base', 'addon', True),
(('dummy', 1), 'add_base', 'addon10', 'unpacked'),
(('dummy', 1), 'add_base', 'addon11', True),
(('dummy', 1), 'add_base', 'addon2', 'unpacked'),
(('dummy', 1), 'add_base', 'addon3', 'unpacked'),
(('dummy', 1), 'add_base', 'addon4', 'unpacked'),
(('dummy', 1), 'add_base', 'addon5', True),
(('dummy', 1), 'add_base', 'addon6', 'unpacked'),
(('dummy', 1), 'add_base', 'addon7', True),
(('dummy', 1), 'add_base', 'addon8', 'unpacked'),
(('dummy', 1), 'add_base', 'addon9', True),
(('dummy', 1), 'add_base', 'hybrid', True),
(('dummy', 1), 'add_base', 'hybrid2', True),
(('dummy', 1), 'add_base', 'qux', False),
(('dummy', 1), 'add_base', 'webextension', True),
((os.path.join(curdir, 'foo', 'bar.manifest'), 2),
'add_manifest', ManifestContent('foo', 'bar', 'bar/')),
((os.path.join(curdir, 'foo', 'bar.manifest'), 1),
'add_manifest', ManifestResource('foo', 'bar', 'bar/')),
(('bar/baz.manifest', 1),
'add_manifest', ManifestResource('bar', 'baz', 'baz/')),
(('qux/qux.manifest', 1),
'add_manifest', ManifestResource('qux', 'qux', 'qux/')),
(('qux/qux.manifest', 2),
'add_manifest', ManifestBinaryComponent('qux', 'qux.so')),
(('manifest', 4), 'add_interfaces', 'foo/bar.xpt', bar_xpt),
(('manifest', 7), 'add_interfaces', 'foo/qux.xpt', qux_xpt),
((os.path.join(curdir, 'addon', 'chrome.manifest'), 1),
'add_manifest', ManifestResource('addon', 'hoge', 'hoge/')),
(('addon2/chrome.manifest', 1), 'add_manifest',
ManifestBinaryComponent('addon2', 'addon2.so')),
(('addon3/components/components.manifest', 1), 'add_manifest',
ManifestBinaryComponent('addon3/components', 'addon3.so')),
(('manifest', 5), 'add', 'foo/bar/foo.html', foo_html),
(('manifest', 5), 'add', 'foo/bar/bar.html', bar_html),
(('manifest', 9), 'add', 'addon/install.rdf', install_rdf),
(('manifest', 10), 'add', 'addon2/install.rdf', install_rdf),
(('manifest', 11), 'add', 'addon3/install.rdf', install_rdf),
(('manifest', 12), 'add', 'addon4/install.rdf',
install_rdf_addon4),
(('manifest', 13), 'add', 'addon5/install.rdf',
install_rdf_addon5),
(('manifest', 14), 'add', 'addon6/install.rdf',
install_rdf_addon6),
(('manifest', 15), 'add', 'addon7/install.rdf',
install_rdf_addon7),
(('manifest', 16), 'add', 'addon8/install.rdf',
install_rdf_addon8),
(('manifest', 17), 'add', 'addon9/install.rdf',
install_rdf_addon9),
(('manifest', 18), 'add', 'addon10/install.rdf',
install_rdf_addon10),
(('manifest', 19), 'add', 'addon11/install.rdf',
install_rdf_addon11),
(('manifest', 20), 'add', 'hybrid/install.rdf', install_rdf),
(('manifest', 21),
'add', 'hybrid/webextension/manifest.json', we_manifest),
(('manifest', 22),
'add', 'hybrid2/webextension/manifest.json', we_manifest),
(('manifest', 23), 'add', 'hybrid2/install.rdf', install_rdf),
(('manifest', 24),
'add', 'webextension/manifest.json', we_manifest),
(('manifest', 25),
'add', 'nonwebextension/manifest.json', non_we_manifest),
])
self.assertEqual(packager.get_bases(),
set(['', 'addon', 'addon2', 'addon3', 'addon4',
'addon5', 'addon6', 'addon7', 'addon8',
'addon9', 'addon10', 'addon11', 'qux',
'hybrid', 'hybrid2', 'webextension']))
self.assertEqual(packager.get_bases(addons=False), set(['', 'qux']))
def repack(source, l10n):
finder = UnpackFinder(source)
l10n_finder = UnpackFinder(l10n)
copier = FileCopier()
if finder.kind == 'flat':
formatter = FlatFormatter(copier)
elif finder.kind == 'jar':
formatter = JarFormatter(copier, optimize=finder.optimizedjars)
elif finder.kind == 'omni':
formatter = OmniJarFormatter(copier, finder.omnijar,
optimize=finder.optimizedjars)
# Read all manifest entries from the packaged directory.
manifests = dict((p, m) for p, m in finder.find('**/*.manifest')
if is_manifest(p))
assert all(isinstance(m, ManifestFile) for m in manifests.itervalues())
entries = [e for m in manifests.itervalues() for e in m if e.localized]
# Find unique locales used in these manifest entries.
locales = list(set(e.id for e in entries if isinstance(e, ManifestLocale)))
# Find all paths whose manifest are included by no other manifest.
includes = set(mozpack.path.join(e.base, e.relpath)
for m in manifests.itervalues()
for e in m if isinstance(e, Manifest))
bases = [mozpack.path.dirname(p) for p in set(manifests.keys()) - includes]
# Read all manifest entries from the langpack directory.
manifests = [m for p, m in l10n_finder.find('**/*.manifest')
if is_manifest(p)]
assert all(isinstance(m, ManifestFile) for m in manifests)
l10n_entries = [e for m in manifests for e in m if e.localized]
# Find unique locales used in these manifest entries.
l10n_locales = list(set(e.id for e in l10n_entries
if isinstance(e, ManifestLocale)))
# The code further below assumes there's only one locale replaced with
# another one.
if len(locales) > 1 or len(l10n_locales) > 1:
errors.fatal("Multiple locales aren't supported")
locale = locales[0]
l10n_locale = l10n_locales[0]
# For each base directory, store what path a locale chrome package name
# corresponds to.
# e.g., for the following entry under app/chrome:
# locale foo en-US path/to/files
# keep track that the locale path for foo in app is
# app/chrome/path/to/files.
l10n_paths = {}
for e in l10n_entries:
if isinstance(e, ManifestChrome):
base = mozpack.path.basedir(e.path, bases)
if not base in l10n_paths:
l10n_paths[base] = {}
l10n_paths[base][e.name] = e.path
# For chrome and non chrome files or directories, store what langpack path
# corresponds to a package path.
paths = dict((e.path,
l10n_paths[mozpack.path.basedir(e.path, bases)][e.name])
for e in entries if isinstance(e, ManifestEntryWithRelPath))
for path in NON_CHROME:
for p, f in l10n_finder.find(path):
paths[p] = p
# Create a new package, with non localized bits coming from the original
# package, and localized bits coming from the langpack.
packager = SimplePackager(formatter)
for p, f in finder:
if is_manifest(p):
# Remove localized manifest entries.
for e in [e for e in f if e.localized]:
f.remove(e)
base = mozpack.path.basedir(p, paths.keys())
if base:
# If the path is one that needs a locale replacement, use the
# corresponding file from the langpack.
subpath = mozpack.path.relpath(p, base)
path = mozpack.path.normpath(mozpack.path.join(paths[base],
subpath))
files = [f for p, f in l10n_finder.find(path)]
if len(files) == 0 and base in NON_CHROME:
path = path.replace(locale, l10n_locale)
files = [f for p, f in l10n_finder.find(path)]
if len(files) == 0:
if not base in NON_CHROME:
errors.error("Missing file: %s" % os.path.join(l10n, path))
else:
packager.add(path, files[0])
else:
packager.add(p, f)
# Add localized manifest entries from the langpack.
l10n_manifests = []
for base in set(e.base for e in l10n_entries):
m = ManifestFile(base)
for e in l10n_entries:
if e.base == base:
m.add(e)
path = mozpack.path.join(base, 'chrome.%s.manifest' % l10n_locale)
l10n_manifests.append((path, m))
bases = packager.get_bases()
for path, m in l10n_manifests:
base = mozpack.path.basedir(path, bases)
packager.add(path, m)
# Add a "manifest $path" entry in the top manifest under that base.
m = ManifestFile(base)
m.add(Manifest(base, mozpack.path.relpath(path, base)))
packager.add(mozpack.path.join(base, 'chrome.manifest'), m)
packager.close()
# Add any remaining non chrome files.
for base in NON_CHROME:
for p, f in l10n_finder.find(base):
if not formatter.contains(p):
formatter.add(p, f)
# Transplant jar preloading information.
for path, log in finder.jarlogs.iteritems():
assert isinstance(copier[path], Jarrer)
copier[path].preload([l.replace(locale, l10n_locale) for l in log])
copier.copy(source)
generate_precomplete(source)
def test_simple_packager(self):
class GeneratedFileWithPath(GeneratedFile):
def __init__(self, path, content):
GeneratedFile.__init__(self, content)
self.path = path
formatter = MockFormatter()
packager = SimplePackager(formatter)
curdir = os.path.abspath(os.curdir)
file = GeneratedFileWithPath(os.path.join(curdir, 'foo',
'bar.manifest'),
'resource bar bar/\ncontent bar bar/')
with errors.context('manifest', 1):
packager.add('foo/bar.manifest', file)
file = GeneratedFileWithPath(os.path.join(curdir, 'foo',
'baz.manifest'),
'resource baz baz/')
with errors.context('manifest', 2):
packager.add('bar/baz.manifest', file)
with errors.context('manifest', 3):
packager.add('qux/qux.manifest',
GeneratedFile('resource qux qux/'))
bar_xpt = GeneratedFile('bar.xpt')
qux_xpt = GeneratedFile('qux.xpt')
foo_html = GeneratedFile('foo_html')
bar_html = GeneratedFile('bar_html')
with errors.context('manifest', 4):
packager.add('foo/bar.xpt', bar_xpt)
with errors.context('manifest', 5):
packager.add('foo/bar/foo.html', foo_html)
packager.add('foo/bar/bar.html', bar_html)
file = GeneratedFileWithPath(os.path.join(curdir, 'foo.manifest'),
''.join([
'manifest foo/bar.manifest\n',
'manifest bar/baz.manifest\n',
]))
with errors.context('manifest', 6):
packager.add('foo.manifest', file)
with errors.context('manifest', 7):
packager.add('foo/qux.xpt', qux_xpt)
self.assertEqual(formatter.log, [])
with errors.context('dummy', 1):
packager.close()
self.maxDiff = None
self.assertEqual(formatter.log, [
(('dummy', 1), 'add_base', 'qux'),
((os.path.join(curdir, 'foo', 'bar.manifest'), 1),
'add_manifest', ManifestResource('foo', 'bar', 'bar/')),
((os.path.join(curdir, 'foo', 'bar.manifest'), 2),
'add_manifest', ManifestContent('foo', 'bar', 'bar/')),
(('bar/baz.manifest', 1),
'add_manifest', ManifestResource('bar', 'baz', 'baz/')),
(('qux/qux.manifest', 1),
'add_manifest', ManifestResource('qux', 'qux', 'qux/')),
(('manifest', 4), 'add_interfaces', 'foo/bar.xpt', bar_xpt),
(('manifest', 7), 'add_interfaces', 'foo/qux.xpt', qux_xpt),
(('manifest', 5), 'add', 'foo/bar/foo.html', foo_html),
(('manifest', 5), 'add', 'foo/bar/bar.html', bar_html),
])
self.assertEqual(packager.get_bases(), set(['', 'qux']))
def test_simple_packager(self):
class GeneratedFileWithPath(GeneratedFile):
def __init__(self, path, content):
GeneratedFile.__init__(self, content)
self.path = path
formatter = MockFormatter()
packager = SimplePackager(formatter)
curdir = os.path.abspath(os.curdir)
file = GeneratedFileWithPath(
os.path.join(curdir, 'foo', 'bar.manifest'),
'resource bar bar/\ncontent bar bar/')
with errors.context('manifest', 1):
packager.add('foo/bar.manifest', file)
file = GeneratedFileWithPath(
os.path.join(curdir, 'foo', 'baz.manifest'), 'resource baz baz/')
with errors.context('manifest', 2):
packager.add('bar/baz.manifest', file)
with errors.context('manifest', 3):
packager.add('qux/qux.manifest',
GeneratedFile('resource qux qux/'))
bar_xpt = GeneratedFile('bar.xpt')
qux_xpt = GeneratedFile('qux.xpt')
foo_html = GeneratedFile('foo_html')
bar_html = GeneratedFile('bar_html')
with errors.context('manifest', 4):
packager.add('foo/bar.xpt', bar_xpt)
with errors.context('manifest', 5):
packager.add('foo/bar/foo.html', foo_html)
packager.add('foo/bar/bar.html', bar_html)
file = GeneratedFileWithPath(
os.path.join(curdir, 'foo.manifest'), ''.join([
'manifest foo/bar.manifest\n',
'manifest bar/baz.manifest\n',
]))
with errors.context('manifest', 6):
packager.add('foo.manifest', file)
with errors.context('manifest', 7):
packager.add('foo/qux.xpt', qux_xpt)
self.assertEqual(formatter.log, [])
with errors.context('dummy', 1):
packager.close()
self.maxDiff = None
self.assertEqual(formatter.log, [
(('dummy', 1), 'add_base', 'qux'),
((os.path.join(curdir, 'foo', 'bar.manifest'), 1), 'add_manifest',
ManifestResource('foo', 'bar', 'bar/')),
((os.path.join(curdir, 'foo', 'bar.manifest'), 2), 'add_manifest',
ManifestContent('foo', 'bar', 'bar/')),
(('bar/baz.manifest', 1), 'add_manifest',
ManifestResource('bar', 'baz', 'baz/')),
(('qux/qux.manifest', 1), 'add_manifest',
ManifestResource('qux', 'qux', 'qux/')),
(('manifest', 4), 'add_interfaces', 'foo/bar.xpt', bar_xpt),
(('manifest', 7), 'add_interfaces', 'foo/qux.xpt', qux_xpt),
(('manifest', 5), 'add', 'foo/bar/foo.html', foo_html),
(('manifest', 5), 'add', 'foo/bar/bar.html', bar_html),
])
self.assertEqual(packager.get_bases(), set(['', 'qux']))
def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()):
app = LocaleManifestFinder(app_finder)
l10n = LocaleManifestFinder(l10n_finder)
# The code further below assumes there's only one locale replaced with
# another one.
if len(app.locales) > 1 or len(l10n.locales) > 1:
errors.fatal("Multiple locales aren't supported")
locale = app.locales[0]
l10n_locale = l10n.locales[0]
# For each base directory, store what path a locale chrome package name
# corresponds to.
# e.g., for the following entry under app/chrome:
# locale foo en-US path/to/files
# keep track that the locale path for foo in app is
# app/chrome/path/to/files.
l10n_paths = {}
for e in l10n.entries:
if isinstance(e, ManifestChrome):
base = mozpack.path.basedir(e.path, app.bases)
l10n_paths.setdefault(base, {})
l10n_paths[base][e.name] = e.path
# For chrome and non chrome files or directories, store what langpack path
# corresponds to a package path.
paths = dict((e.path,
l10n_paths[mozpack.path.basedir(e.path, app.bases)][e.name])
for e in app.entries
if isinstance(e, ManifestEntryWithRelPath))
for pattern in non_chrome:
for base in app.bases:
path = mozpack.path.join(base, pattern)
left = set(p for p, f in app_finder.find(path))
right = set(p for p, f in l10n_finder.find(path))
for p in right:
paths[p] = p
for p in left - right:
paths[p] = None
# Create a new package, with non localized bits coming from the original
# package, and localized bits coming from the langpack.
packager = SimplePackager(formatter)
for p, f in app_finder:
if is_manifest(p):
# Remove localized manifest entries.
for e in [e for e in f if e.localized]:
f.remove(e)
# If the path is one that needs a locale replacement, use the
# corresponding file from the langpack.
path = None
if p in paths:
path = paths[p]
if not path:
continue
else:
base = mozpack.path.basedir(p, paths.keys())
if base:
subpath = mozpack.path.relpath(p, base)
path = mozpack.path.normpath(mozpack.path.join(paths[base],
subpath))
if path:
files = [f for p, f in l10n_finder.find(path)]
if not len(files):
if base not in non_chrome:
errors.error("Missing file: %s" %
os.path.join(l10n_finder.base, path))
else:
packager.add(path, files[0])
else:
packager.add(p, f)
# Add localized manifest entries from the langpack.
l10n_manifests = []
for base in set(e.base for e in l10n.entries):
m = ManifestFile(base, [e for e in l10n.entries if e.base == base])
path = mozpack.path.join(base, 'chrome.%s.manifest' % l10n_locale)
l10n_manifests.append((path, m))
bases = packager.get_bases()
for path, m in l10n_manifests:
base = mozpack.path.basedir(path, bases)
packager.add(path, m)
# Add a "manifest $path" entry in the top manifest under that base.
m = ManifestFile(base)
m.add(Manifest(base, mozpack.path.relpath(path, base)))
packager.add(mozpack.path.join(base, 'chrome.manifest'), m)
packager.close()
# Add any remaining non chrome files.
for pattern in non_chrome:
for base in bases:
for p, f in l10n_finder.find(mozpack.path.join(base, pattern)):
if not formatter.contains(p):
formatter.add(p, f)
# Transplant jar preloading information.
for path, log in app_finder.jarlogs.iteritems():
assert isinstance(copier[path], Jarrer)
copier[path].preload([l.replace(locale, l10n_locale) for l in log])
