def unpack_to_registry(source, registry): ''' Transform a jar chrome or omnijar packaged directory into a flat package. The given registry is filled with the flat package. ''' finder = UnpackFinder(source) packager = SimplePackager(FlatFormatter(registry)) for p, f in finder.find('*'): packager.add(p, f) packager.close()
def unpack(source): ''' Transform a jar chrome or omnijar packaged directory into a flat package. ''' copier = FileCopier() finder = UnpackFinder(source) packager = SimplePackager(FlatFormatter(copier)) for p, f in finder.find('*'): if mozpack.path.split(p)[0] not in STARTUP_CACHE_PATHS: packager.add(p, f) packager.close() copier.copy(source, skip_if_older=False)
def unpack_to_registry(source, registry): ''' Transform a jar chrome or omnijar packaged directory into a flat package. The given registry is filled with the flat package. ''' finder = UnpackFinder(source) packager = SimplePackager(FlatFormatter(registry)) for p, f in finder.find('*'): if mozpath.split(p)[0] not in STARTUP_CACHE_PATHS: packager.add(p, f) packager.close()
def unpack_to_registry(source, registry, omnijar_name=None): """ Transform a jar chrome or omnijar packaged directory into a flat package. The given registry is filled with the flat package. """ finder = UnpackFinder(source, omnijar_name) packager = SimplePackager(FlatFormatter(registry)) for p, f in finder.find("*"): packager.add(p, f) packager.close()
def main(): parser = argparse.ArgumentParser( description= "Merge two builds of a Gecko-based application into a Universal build") parser.add_argument("app1", help="Directory containing the application") parser.add_argument("app2", help="Directory containing the application to merge") parser.add_argument( "--non-resource", nargs="+", metavar="PATTERN", default=[], help="Extra files not to be considered as resources", ) options = parser.parse_args() buildconfig.substs["OS_ARCH"] = "Darwin" buildconfig.substs["LIPO"] = os.environ.get("LIPO") app1_finder = UnpackFinder(FileFinder(options.app1, find_executables=True)) app2_finder = UnpackFinder(FileFinder(options.app2, find_executables=True)) app_finder = UnifiedBuildFinderWasmHack(app1_finder, app2_finder) copier = FileCopier() compress = min(app1_finder.compressed, JAR_DEFLATED) if app1_finder.kind == "flat": formatter = FlatFormatter(copier) elif app1_finder.kind == "jar": formatter = JarFormatter(copier, compress=compress) elif app1_finder.kind == "omni": formatter = OmniJarFormatter( copier, app1_finder.omnijar, compress=compress, non_resources=options.non_resource, ) with errors.accumulate(): packager = SimplePackager(formatter) for p, f in app_finder: packager.add(p, f) packager.close() # Transplant jar preloading information. for path, log in six.iteritems(app1_finder.jarlogs): assert isinstance(copier[path], Jarrer) copier[path].preload(log) copier.copy(options.app1, skip_if_older=False)
def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()): app = LocaleManifestFinder(app_finder) l10n = LocaleManifestFinder(l10n_finder) # The code further below assumes there's only one locale replaced with # another one. if len(app.locales) > 1: errors.fatal("Multiple app locales aren't supported: " + ",".join(app.locales)) if len(l10n.locales) > 1: errors.fatal("Multiple l10n locales aren't supported: " + ",".join(l10n.locales)) locale = app.locales[0] l10n_locale = l10n.locales[0] # For each base directory, store what path a locale chrome package name # corresponds to. # e.g., for the following entry under app/chrome: # locale foo en-US path/to/files # keep track that the locale path for foo in app is # app/chrome/path/to/files. # As there may be multiple locale entries with the same base, but with # different flags, that tracking takes the flags into account when there # are some. Example: # locale foo en-US path/to/files/win os=Win # locale foo en-US path/to/files/mac os=Darwin def key(entry): if entry.flags: return '%s %s' % (entry.name, entry.flags) return entry.name l10n_paths = {} for e in l10n.entries: if isinstance(e, ManifestChrome): base = mozpath.basedir(e.path, app.bases) l10n_paths.setdefault(base, {}) l10n_paths[base][key(e)] = e.path # For chrome and non chrome files or directories, store what langpack path # corresponds to a package path. paths = {} for e in app.entries: if isinstance(e, ManifestEntryWithRelPath): base = mozpath.basedir(e.path, app.bases) if base not in l10n_paths: errors.fatal("Locale doesn't contain %s/" % base) # Allow errors to accumulate continue if key(e) not in l10n_paths[base]: errors.fatal("Locale doesn't have a manifest entry for '%s'" % e.name) # Allow errors to accumulate continue paths[e.path] = l10n_paths[base][key(e)] for pattern in non_chrome: for base in app.bases: path = mozpath.join(base, pattern) left = set(p for p, f in app_finder.find(path)) right = set(p for p, f in l10n_finder.find(path)) for p in right: paths[p] = p for p in left - right: paths[p] = None # Create a new package, with non localized bits coming from the original # package, and localized bits coming from the langpack. packager = SimplePackager(formatter) for p, f in app_finder: if is_manifest(p): # Remove localized manifest entries. for e in [e for e in f if e.localized]: f.remove(e) # If the path is one that needs a locale replacement, use the # corresponding file from the langpack. path = None if p in paths: path = paths[p] if not path: continue else: base = mozpath.basedir(p, paths.keys()) if base: subpath = mozpath.relpath(p, base) path = mozpath.normpath(mozpath.join(paths[base], subpath)) if path: files = [f for p, f in l10n_finder.find(path)] if not len(files): if base not in non_chrome: finderBase = "" if hasattr(l10n_finder, 'base'): finderBase = l10n_finder.base errors.error("Missing file: %s" % os.path.join(finderBase, path)) else: packager.add(path, files[0]) else: packager.add(p, f) # Add localized manifest entries from the langpack. l10n_manifests = [] for base in set(e.base for e in l10n.entries): m = ManifestFile(base, [e for e in l10n.entries if e.base == base]) path = mozpath.join(base, 'chrome.%s.manifest' % l10n_locale) l10n_manifests.append((path, m)) bases = packager.get_bases() for path, m in l10n_manifests: base = mozpath.basedir(path, bases) packager.add(path, m) # Add a "manifest $path" entry in the top manifest under that base. m = ManifestFile(base) m.add(Manifest(base, mozpath.relpath(path, base))) packager.add(mozpath.join(base, 'chrome.manifest'), m) packager.close() # Add any remaining non chrome files. for pattern in non_chrome: for base in bases: for p, f in l10n_finder.find(mozpath.join(base, pattern)): if not formatter.contains(p): formatter.add(p, f) # Resources in `localization` directories are packaged from the source and then # if localized versions are present in the l10n dir, we package them as well # keeping the source dir resources as a runtime fallback. for p, f in l10n_finder.find('**/localization'): if not formatter.contains(p): formatter.add(p, f) # Transplant jar preloading information. for path, log in six.iteritems(app_finder.jarlogs): assert isinstance(copier[path], Jarrer) copier[path].preload([l.replace(locale, l10n_locale) for l in log])
def test_simple_packager_manifest_consistency(self): formatter = MockFormatter() # bar/ is detected as an addon because of install.rdf, but top-level # includes a manifest inside bar/. packager = SimplePackager(formatter) packager.add('base.manifest', GeneratedFile( 'manifest foo/bar.manifest\n' 'manifest bar/baz.manifest\n' )) packager.add('foo/bar.manifest', GeneratedFile('resource bar bar')) packager.add('bar/baz.manifest', GeneratedFile('resource baz baz')) packager.add('bar/install.rdf', GeneratedFile('')) with self.assertRaises(ErrorMessage) as e: packager.close() self.assertEqual(e.exception.message, 'Error: "bar/baz.manifest" is included from "base.manifest", ' 'which is outside "bar"') # bar/ is detected as a separate base because of chrome.manifest that # is included nowhere, but top-level includes another manifest inside # bar/. packager = SimplePackager(formatter) packager.add('base.manifest', GeneratedFile( 'manifest foo/bar.manifest\n' 'manifest bar/baz.manifest\n' )) packager.add('foo/bar.manifest', GeneratedFile('resource bar bar')) packager.add('bar/baz.manifest', GeneratedFile('resource baz baz')) packager.add('bar/chrome.manifest', GeneratedFile('resource baz baz')) with self.assertRaises(ErrorMessage) as e: packager.close() self.assertEqual(e.exception.message, 'Error: "bar/baz.manifest" is included from "base.manifest", ' 'which is outside "bar"') # bar/ is detected as a separate base because of chrome.manifest that # is included nowhere, but chrome.manifest includes baz.manifest from # the same directory. This shouldn't error out. packager = SimplePackager(formatter) packager.add('base.manifest', GeneratedFile( 'manifest foo/bar.manifest\n' )) packager.add('foo/bar.manifest', GeneratedFile('resource bar bar')) packager.add('bar/baz.manifest', GeneratedFile('resource baz baz')) packager.add('bar/chrome.manifest', GeneratedFile('manifest baz.manifest')) packager.close()
def test_simple_packager(self): class GeneratedFileWithPath(GeneratedFile): def __init__(self, path, content): GeneratedFile.__init__(self, content) self.path = path formatter = MockFormatter() packager = SimplePackager(formatter) curdir = os.path.abspath(os.curdir) file = GeneratedFileWithPath(os.path.join(curdir, 'foo', 'bar.manifest'), 'resource bar bar/\ncontent bar bar/') with errors.context('manifest', 1): packager.add('foo/bar.manifest', file) file = GeneratedFileWithPath(os.path.join(curdir, 'foo', 'baz.manifest'), 'resource baz baz/') with errors.context('manifest', 2): packager.add('bar/baz.manifest', file) with errors.context('manifest', 3): packager.add('qux/qux.manifest', GeneratedFile('resource qux qux/')) bar_xpt = GeneratedFile('bar.xpt') qux_xpt = GeneratedFile('qux.xpt') foo_html = GeneratedFile('foo_html') bar_html = GeneratedFile('bar_html') with errors.context('manifest', 4): packager.add('foo/bar.xpt', bar_xpt) with errors.context('manifest', 5): packager.add('foo/bar/foo.html', foo_html) packager.add('foo/bar/bar.html', bar_html) file = GeneratedFileWithPath(os.path.join(curdir, 'foo.manifest'), ''.join([ 'manifest foo/bar.manifest\n', 'manifest bar/baz.manifest\n', ])) with errors.context('manifest', 6): packager.add('foo.manifest', file) with errors.context('manifest', 7): packager.add('foo/qux.xpt', qux_xpt) file = GeneratedFileWithPath(os.path.join(curdir, 'addon', 'chrome.manifest'), 'resource hoge hoge/') with errors.context('manifest', 8): packager.add('addon/chrome.manifest', file) install_rdf = GeneratedFile('<RDF></RDF>') with errors.context('manifest', 9): packager.add('addon/install.rdf', install_rdf) self.assertEqual(formatter.log, []) with errors.context('dummy', 1): packager.close() self.maxDiff = None # The formatter is expected to reorder the manifest entries so that # chrome entries appear before the others. self.assertEqual(formatter.log, [ (('dummy', 1), 'add_base', 'qux', False), (('dummy', 1), 'add_base', 'addon', True), ((os.path.join(curdir, 'foo', 'bar.manifest'), 2), 'add_manifest', ManifestContent('foo', 'bar', 'bar/')), ((os.path.join(curdir, 'foo', 'bar.manifest'), 1), 'add_manifest', ManifestResource('foo', 'bar', 'bar/')), (('bar/baz.manifest', 1), 'add_manifest', ManifestResource('bar', 'baz', 'baz/')), (('qux/qux.manifest', 1), 'add_manifest', ManifestResource('qux', 'qux', 'qux/')), (('manifest', 4), 'add_interfaces', 'foo/bar.xpt', bar_xpt), (('manifest', 7), 'add_interfaces', 'foo/qux.xpt', qux_xpt), ((os.path.join(curdir, 'addon', 'chrome.manifest'), 1), 'add_manifest', ManifestResource('addon', 'hoge', 'hoge/')), (('manifest', 5), 'add', 'foo/bar/foo.html', foo_html), (('manifest', 5), 'add', 'foo/bar/bar.html', bar_html), (('manifest', 9), 'add', 'addon/install.rdf', install_rdf), ]) self.assertEqual(packager.get_bases(), set(['', 'addon', 'qux'])) self.assertEqual(packager.get_bases(addons=False), set(['', 'qux']))
def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()): app = LocaleManifestFinder(app_finder) l10n = LocaleManifestFinder(l10n_finder) # The code further below assumes there's only one locale replaced with # another one. if len(app.locales) > 1 or len(l10n.locales) > 1: errors.fatal("Multiple locales aren't supported") locale = app.locales[0] l10n_locale = l10n.locales[0] # For each base directory, store what path a locale chrome package name # corresponds to. # e.g., for the following entry under app/chrome: # locale foo en-US path/to/files # keep track that the locale path for foo in app is # app/chrome/path/to/files. l10n_paths = {} for e in l10n.entries: if isinstance(e, ManifestChrome): base = mozpath.basedir(e.path, app.bases) l10n_paths.setdefault(base, {}) l10n_paths[base][e.name] = e.path # For chrome and non chrome files or directories, store what langpack path # corresponds to a package path. paths = {} for e in app.entries: if isinstance(e, ManifestEntryWithRelPath): base = mozpath.basedir(e.path, app.bases) if base not in l10n_paths: errors.fatal("Locale doesn't contain %s/" % base) # Allow errors to accumulate continue if e.name not in l10n_paths[base]: errors.fatal("Locale doesn't have a manifest entry for '%s'" % e.name) # Allow errors to accumulate continue paths[e.path] = l10n_paths[base][e.name] for pattern in non_chrome: for base in app.bases: path = mozpath.join(base, pattern) left = set(p for p, f in app_finder.find(path)) right = set(p for p, f in l10n_finder.find(path)) for p in right: paths[p] = p for p in left - right: paths[p] = None # Create a new package, with non localized bits coming from the original # package, and localized bits coming from the langpack. packager = SimplePackager(formatter) for p, f in app_finder: if is_manifest(p): # Remove localized manifest entries. for e in [e for e in f if e.localized]: f.remove(e) # If the path is one that needs a locale replacement, use the # corresponding file from the langpack. path = None if p in paths: path = paths[p] if not path: continue else: base = mozpath.basedir(p, paths.keys()) if base: subpath = mozpath.relpath(p, base) path = mozpath.normpath(mozpath.join(paths[base], subpath)) if path: files = [f for p, f in l10n_finder.find(path)] if not len(files): if base not in non_chrome: errors.error("Missing file: %s" % os.path.join(l10n_finder.base, path)) else: packager.add(path, files[0]) else: packager.add(p, f) # Add localized manifest entries from the langpack. l10n_manifests = [] for base in set(e.base for e in l10n.entries): m = ManifestFile(base, [e for e in l10n.entries if e.base == base]) path = mozpath.join(base, 'chrome.%s.manifest' % l10n_locale) l10n_manifests.append((path, m)) bases = packager.get_bases() for path, m in l10n_manifests: base = mozpath.basedir(path, bases) packager.add(path, m) # Add a "manifest $path" entry in the top manifest under that base. m = ManifestFile(base) m.add(Manifest(base, mozpath.relpath(path, base))) packager.add(mozpath.join(base, 'chrome.manifest'), m) packager.close() # Add any remaining non chrome files. for pattern in non_chrome: for base in bases: for p, f in l10n_finder.find(mozpath.join(base, pattern)): if not formatter.contains(p): formatter.add(p, f) # Transplant jar preloading information. for path, log in app_finder.jarlogs.iteritems(): assert isinstance(copier[path], Jarrer) copier[path].preload([l.replace(locale, l10n_locale) for l in log])
def test_simple_packager(self): class GeneratedFileWithPath(GeneratedFile): def __init__(self, path, content): GeneratedFile.__init__(self, content) self.path = path formatter = MockFormatter() packager = SimplePackager(formatter) curdir = os.path.abspath(os.curdir) file = GeneratedFileWithPath(os.path.join(curdir, 'foo', 'bar.manifest'), 'resource bar bar/\ncontent bar bar/') with errors.context('manifest', 1): packager.add('foo/bar.manifest', file) file = GeneratedFileWithPath(os.path.join(curdir, 'foo', 'baz.manifest'), 'resource baz baz/') with errors.context('manifest', 2): packager.add('bar/baz.manifest', file) with errors.context('manifest', 3): packager.add('qux/qux.manifest', GeneratedFile(''.join([ 'resource qux qux/\n', 'binary-component qux.so\n', ]))) bar_xpt = GeneratedFile('bar.xpt') qux_xpt = GeneratedFile('qux.xpt') foo_html = GeneratedFile('foo_html') bar_html = GeneratedFile('bar_html') with errors.context('manifest', 4): packager.add('foo/bar.xpt', bar_xpt) with errors.context('manifest', 5): packager.add('foo/bar/foo.html', foo_html) packager.add('foo/bar/bar.html', bar_html) file = GeneratedFileWithPath(os.path.join(curdir, 'foo.manifest'), ''.join([ 'manifest foo/bar.manifest\n', 'manifest bar/baz.manifest\n', ])) with errors.context('manifest', 6): packager.add('foo.manifest', file) with errors.context('manifest', 7): packager.add('foo/qux.xpt', qux_xpt) file = GeneratedFileWithPath(os.path.join(curdir, 'addon', 'chrome.manifest'), 'resource hoge hoge/') with errors.context('manifest', 8): packager.add('addon/chrome.manifest', file) install_rdf = GeneratedFile('<RDF></RDF>') with errors.context('manifest', 9): packager.add('addon/install.rdf', install_rdf) with errors.context('manifest', 10): packager.add('addon2/install.rdf', install_rdf) packager.add('addon2/chrome.manifest', GeneratedFile('binary-component addon2.so')) with errors.context('manifest', 11): packager.add('addon3/install.rdf', install_rdf) packager.add('addon3/chrome.manifest', GeneratedFile( 'manifest components/components.manifest')) packager.add('addon3/components/components.manifest', GeneratedFile('binary-component addon3.so')) with errors.context('manifest', 12): install_rdf_addon4 = GeneratedFile( '<RDF>\n<...>\n<em:unpack>true</em:unpack>\n<...>\n</RDF>') packager.add('addon4/install.rdf', install_rdf_addon4) with errors.context('manifest', 13): install_rdf_addon5 = GeneratedFile( '<RDF>\n<...>\n<em:unpack>false</em:unpack>\n<...>\n</RDF>') packager.add('addon5/install.rdf', install_rdf_addon5) with errors.context('manifest', 14): install_rdf_addon6 = GeneratedFile( '<RDF>\n<... em:unpack=true>\n<...>\n</RDF>') packager.add('addon6/install.rdf', install_rdf_addon6) with errors.context('manifest', 15): install_rdf_addon7 = GeneratedFile( '<RDF>\n<... em:unpack=false>\n<...>\n</RDF>') packager.add('addon7/install.rdf', install_rdf_addon7) with errors.context('manifest', 16): install_rdf_addon8 = GeneratedFile( '<RDF>\n<... em:unpack="true">\n<...>\n</RDF>') packager.add('addon8/install.rdf', install_rdf_addon8) with errors.context('manifest', 17): install_rdf_addon9 = GeneratedFile( '<RDF>\n<... em:unpack="false">\n<...>\n</RDF>') packager.add('addon9/install.rdf', install_rdf_addon9) with errors.context('manifest', 18): install_rdf_addon10 = GeneratedFile( '<RDF>\n<... em:unpack=\'true\'>\n<...>\n</RDF>') packager.add('addon10/install.rdf', install_rdf_addon10) with errors.context('manifest', 19): install_rdf_addon11 = GeneratedFile( '<RDF>\n<... em:unpack=\'false\'>\n<...>\n</RDF>') packager.add('addon11/install.rdf', install_rdf_addon11) self.assertEqual(formatter.log, []) with errors.context('dummy', 1): packager.close() self.maxDiff = None # The formatter is expected to reorder the manifest entries so that # chrome entries appear before the others. self.assertEqual(formatter.log, [ (('dummy', 1), 'add_base', '', False), (('dummy', 1), 'add_base', 'addon', True), (('dummy', 1), 'add_base', 'addon10', 'unpacked'), (('dummy', 1), 'add_base', 'addon11', True), (('dummy', 1), 'add_base', 'addon2', 'unpacked'), (('dummy', 1), 'add_base', 'addon3', 'unpacked'), (('dummy', 1), 'add_base', 'addon4', 'unpacked'), (('dummy', 1), 'add_base', 'addon5', True), (('dummy', 1), 'add_base', 'addon6', 'unpacked'), (('dummy', 1), 'add_base', 'addon7', True), (('dummy', 1), 'add_base', 'addon8', 'unpacked'), (('dummy', 1), 'add_base', 'addon9', True), (('dummy', 1), 'add_base', 'qux', False), ((os.path.join(curdir, 'foo', 'bar.manifest'), 2), 'add_manifest', ManifestContent('foo', 'bar', 'bar/')), ((os.path.join(curdir, 'foo', 'bar.manifest'), 1), 'add_manifest', ManifestResource('foo', 'bar', 'bar/')), (('bar/baz.manifest', 1), 'add_manifest', ManifestResource('bar', 'baz', 'baz/')), (('qux/qux.manifest', 1), 'add_manifest', ManifestResource('qux', 'qux', 'qux/')), (('qux/qux.manifest', 2), 'add_manifest', ManifestBinaryComponent('qux', 'qux.so')), (('manifest', 4), 'add_interfaces', 'foo/bar.xpt', bar_xpt), (('manifest', 7), 'add_interfaces', 'foo/qux.xpt', qux_xpt), ((os.path.join(curdir, 'addon', 'chrome.manifest'), 1), 'add_manifest', ManifestResource('addon', 'hoge', 'hoge/')), (('addon2/chrome.manifest', 1), 'add_manifest', ManifestBinaryComponent('addon2', 'addon2.so')), (('addon3/components/components.manifest', 1), 'add_manifest', ManifestBinaryComponent('addon3/components', 'addon3.so')), (('manifest', 5), 'add', 'foo/bar/foo.html', foo_html), (('manifest', 5), 'add', 'foo/bar/bar.html', bar_html), (('manifest', 9), 'add', 'addon/install.rdf', install_rdf), (('manifest', 10), 'add', 'addon2/install.rdf', install_rdf), (('manifest', 11), 'add', 'addon3/install.rdf', install_rdf), (('manifest', 12), 'add', 'addon4/install.rdf', install_rdf_addon4), (('manifest', 13), 'add', 'addon5/install.rdf', install_rdf_addon5), (('manifest', 14), 'add', 'addon6/install.rdf', install_rdf_addon6), (('manifest', 15), 'add', 'addon7/install.rdf', install_rdf_addon7), (('manifest', 16), 'add', 'addon8/install.rdf', install_rdf_addon8), (('manifest', 17), 'add', 'addon9/install.rdf', install_rdf_addon9), (('manifest', 18), 'add', 'addon10/install.rdf', install_rdf_addon10), (('manifest', 19), 'add', 'addon11/install.rdf', install_rdf_addon11), ]) self.assertEqual(packager.get_bases(), set(['', 'addon', 'addon2', 'addon3', 'addon4', 'addon5', 'addon6', 'addon7', 'addon8', 'addon9', 'addon10', 'addon11', 'qux'])) self.assertEqual(packager.get_bases(addons=False), set(['', 'qux']))
def test_simple_packager(self): class GeneratedFileWithPath(GeneratedFile): def __init__(self, path, content): GeneratedFile.__init__(self, content) self.path = path formatter = MockFormatter() packager = SimplePackager(formatter) curdir = os.path.abspath(os.curdir) file = GeneratedFileWithPath(os.path.join(curdir, 'foo', 'bar.manifest'), 'resource bar bar/\ncontent bar bar/') with errors.context('manifest', 1): packager.add('foo/bar.manifest', file) file = GeneratedFileWithPath(os.path.join(curdir, 'foo', 'baz.manifest'), 'resource baz baz/') with errors.context('manifest', 2): packager.add('bar/baz.manifest', file) with errors.context('manifest', 3): packager.add('qux/qux.manifest', GeneratedFile(''.join([ 'resource qux qux/\n', 'binary-component qux.so\n', ]))) bar_xpt = GeneratedFile('bar.xpt') qux_xpt = GeneratedFile('qux.xpt') foo_html = GeneratedFile('foo_html') bar_html = GeneratedFile('bar_html') with errors.context('manifest', 4): packager.add('foo/bar.xpt', bar_xpt) with errors.context('manifest', 5): packager.add('foo/bar/foo.html', foo_html) packager.add('foo/bar/bar.html', bar_html) file = GeneratedFileWithPath(os.path.join(curdir, 'foo.manifest'), ''.join([ 'manifest foo/bar.manifest\n', 'manifest bar/baz.manifest\n', ])) with errors.context('manifest', 6): packager.add('foo.manifest', file) with errors.context('manifest', 7): packager.add('foo/qux.xpt', qux_xpt) file = GeneratedFileWithPath(os.path.join(curdir, 'addon', 'chrome.manifest'), 'resource hoge hoge/') with errors.context('manifest', 8): packager.add('addon/chrome.manifest', file) install_rdf = GeneratedFile('<RDF></RDF>') with errors.context('manifest', 9): packager.add('addon/install.rdf', install_rdf) with errors.context('manifest', 10): packager.add('addon2/install.rdf', install_rdf) packager.add('addon2/chrome.manifest', GeneratedFile('binary-component addon2.so')) with errors.context('manifest', 11): packager.add('addon3/install.rdf', install_rdf) packager.add('addon3/chrome.manifest', GeneratedFile( 'manifest components/components.manifest')) packager.add('addon3/components/components.manifest', GeneratedFile('binary-component addon3.so')) with errors.context('manifest', 12): install_rdf_addon4 = GeneratedFile( '<RDF>\n<...>\n<em:unpack>true</em:unpack>\n<...>\n</RDF>') packager.add('addon4/install.rdf', install_rdf_addon4) with errors.context('manifest', 13): install_rdf_addon5 = GeneratedFile( '<RDF>\n<...>\n<em:unpack>false</em:unpack>\n<...>\n</RDF>') packager.add('addon5/install.rdf', install_rdf_addon5) with errors.context('manifest', 14): install_rdf_addon6 = GeneratedFile( '<RDF>\n<... em:unpack=true>\n<...>\n</RDF>') packager.add('addon6/install.rdf', install_rdf_addon6) with errors.context('manifest', 15): install_rdf_addon7 = GeneratedFile( '<RDF>\n<... em:unpack=false>\n<...>\n</RDF>') packager.add('addon7/install.rdf', install_rdf_addon7) with errors.context('manifest', 16): install_rdf_addon8 = GeneratedFile( '<RDF>\n<... em:unpack="true">\n<...>\n</RDF>') packager.add('addon8/install.rdf', install_rdf_addon8) with errors.context('manifest', 17): install_rdf_addon9 = GeneratedFile( '<RDF>\n<... em:unpack="false">\n<...>\n</RDF>') packager.add('addon9/install.rdf', install_rdf_addon9) with errors.context('manifest', 18): install_rdf_addon10 = GeneratedFile( '<RDF>\n<... em:unpack=\'true\'>\n<...>\n</RDF>') packager.add('addon10/install.rdf', install_rdf_addon10) with errors.context('manifest', 19): install_rdf_addon11 = GeneratedFile( '<RDF>\n<... em:unpack=\'false\'>\n<...>\n</RDF>') packager.add('addon11/install.rdf', install_rdf_addon11) we_manifest = GeneratedFile( '{"manifest_version": 2, "name": "Test WebExtension", "version": "1.0"}') # hybrid and hybrid2 are both bootstrapped extensions with # embedded webextensions, they differ in the order in which # the manifests are added to the packager. with errors.context('manifest', 20): packager.add('hybrid/install.rdf', install_rdf) with errors.context('manifest', 21): packager.add('hybrid/webextension/manifest.json', we_manifest) with errors.context('manifest', 22): packager.add('hybrid2/webextension/manifest.json', we_manifest) with errors.context('manifest', 23): packager.add('hybrid2/install.rdf', install_rdf) with errors.context('manifest', 24): packager.add('webextension/manifest.json', we_manifest) non_we_manifest = GeneratedFile('{"not a webextension": true}') with errors.context('manifest', 25): packager.add('nonwebextension/manifest.json', non_we_manifest) self.assertEqual(formatter.log, []) with errors.context('dummy', 1): packager.close() self.maxDiff = None # The formatter is expected to reorder the manifest entries so that # chrome entries appear before the others. self.assertEqual(formatter.log, [ (('dummy', 1), 'add_base', '', False), (('dummy', 1), 'add_base', 'addon', True), (('dummy', 1), 'add_base', 'addon10', 'unpacked'), (('dummy', 1), 'add_base', 'addon11', True), (('dummy', 1), 'add_base', 'addon2', 'unpacked'), (('dummy', 1), 'add_base', 'addon3', 'unpacked'), (('dummy', 1), 'add_base', 'addon4', 'unpacked'), (('dummy', 1), 'add_base', 'addon5', True), (('dummy', 1), 'add_base', 'addon6', 'unpacked'), (('dummy', 1), 'add_base', 'addon7', True), (('dummy', 1), 'add_base', 'addon8', 'unpacked'), (('dummy', 1), 'add_base', 'addon9', True), (('dummy', 1), 'add_base', 'hybrid', True), (('dummy', 1), 'add_base', 'hybrid2', True), (('dummy', 1), 'add_base', 'qux', False), (('dummy', 1), 'add_base', 'webextension', True), ((os.path.join(curdir, 'foo', 'bar.manifest'), 2), 'add_manifest', ManifestContent('foo', 'bar', 'bar/')), ((os.path.join(curdir, 'foo', 'bar.manifest'), 1), 'add_manifest', ManifestResource('foo', 'bar', 'bar/')), (('bar/baz.manifest', 1), 'add_manifest', ManifestResource('bar', 'baz', 'baz/')), (('qux/qux.manifest', 1), 'add_manifest', ManifestResource('qux', 'qux', 'qux/')), (('qux/qux.manifest', 2), 'add_manifest', ManifestBinaryComponent('qux', 'qux.so')), (('manifest', 4), 'add_interfaces', 'foo/bar.xpt', bar_xpt), (('manifest', 7), 'add_interfaces', 'foo/qux.xpt', qux_xpt), ((os.path.join(curdir, 'addon', 'chrome.manifest'), 1), 'add_manifest', ManifestResource('addon', 'hoge', 'hoge/')), (('addon2/chrome.manifest', 1), 'add_manifest', ManifestBinaryComponent('addon2', 'addon2.so')), (('addon3/components/components.manifest', 1), 'add_manifest', ManifestBinaryComponent('addon3/components', 'addon3.so')), (('manifest', 5), 'add', 'foo/bar/foo.html', foo_html), (('manifest', 5), 'add', 'foo/bar/bar.html', bar_html), (('manifest', 9), 'add', 'addon/install.rdf', install_rdf), (('manifest', 10), 'add', 'addon2/install.rdf', install_rdf), (('manifest', 11), 'add', 'addon3/install.rdf', install_rdf), (('manifest', 12), 'add', 'addon4/install.rdf', install_rdf_addon4), (('manifest', 13), 'add', 'addon5/install.rdf', install_rdf_addon5), (('manifest', 14), 'add', 'addon6/install.rdf', install_rdf_addon6), (('manifest', 15), 'add', 'addon7/install.rdf', install_rdf_addon7), (('manifest', 16), 'add', 'addon8/install.rdf', install_rdf_addon8), (('manifest', 17), 'add', 'addon9/install.rdf', install_rdf_addon9), (('manifest', 18), 'add', 'addon10/install.rdf', install_rdf_addon10), (('manifest', 19), 'add', 'addon11/install.rdf', install_rdf_addon11), (('manifest', 20), 'add', 'hybrid/install.rdf', install_rdf), (('manifest', 21), 'add', 'hybrid/webextension/manifest.json', we_manifest), (('manifest', 22), 'add', 'hybrid2/webextension/manifest.json', we_manifest), (('manifest', 23), 'add', 'hybrid2/install.rdf', install_rdf), (('manifest', 24), 'add', 'webextension/manifest.json', we_manifest), (('manifest', 25), 'add', 'nonwebextension/manifest.json', non_we_manifest), ]) self.assertEqual(packager.get_bases(), set(['', 'addon', 'addon2', 'addon3', 'addon4', 'addon5', 'addon6', 'addon7', 'addon8', 'addon9', 'addon10', 'addon11', 'qux', 'hybrid', 'hybrid2', 'webextension'])) self.assertEqual(packager.get_bases(addons=False), set(['', 'qux']))
def repack(source, l10n): finder = UnpackFinder(source) l10n_finder = UnpackFinder(l10n) copier = FileCopier() if finder.kind == 'flat': formatter = FlatFormatter(copier) elif finder.kind == 'jar': formatter = JarFormatter(copier, optimize=finder.optimizedjars) elif finder.kind == 'omni': formatter = OmniJarFormatter(copier, finder.omnijar, optimize=finder.optimizedjars) # Read all manifest entries from the packaged directory. manifests = dict((p, m) for p, m in finder.find('**/*.manifest') if is_manifest(p)) assert all(isinstance(m, ManifestFile) for m in manifests.itervalues()) entries = [e for m in manifests.itervalues() for e in m if e.localized] # Find unique locales used in these manifest entries. locales = list(set(e.id for e in entries if isinstance(e, ManifestLocale))) # Find all paths whose manifest are included by no other manifest. includes = set(mozpack.path.join(e.base, e.relpath) for m in manifests.itervalues() for e in m if isinstance(e, Manifest)) bases = [mozpack.path.dirname(p) for p in set(manifests.keys()) - includes] # Read all manifest entries from the langpack directory. manifests = [m for p, m in l10n_finder.find('**/*.manifest') if is_manifest(p)] assert all(isinstance(m, ManifestFile) for m in manifests) l10n_entries = [e for m in manifests for e in m if e.localized] # Find unique locales used in these manifest entries. l10n_locales = list(set(e.id for e in l10n_entries if isinstance(e, ManifestLocale))) # The code further below assumes there's only one locale replaced with # another one. if len(locales) > 1 or len(l10n_locales) > 1: errors.fatal("Multiple locales aren't supported") locale = locales[0] l10n_locale = l10n_locales[0] # For each base directory, store what path a locale chrome package name # corresponds to. # e.g., for the following entry under app/chrome: # locale foo en-US path/to/files # keep track that the locale path for foo in app is # app/chrome/path/to/files. l10n_paths = {} for e in l10n_entries: if isinstance(e, ManifestChrome): base = mozpack.path.basedir(e.path, bases) if not base in l10n_paths: l10n_paths[base] = {} l10n_paths[base][e.name] = e.path # For chrome and non chrome files or directories, store what langpack path # corresponds to a package path. paths = dict((e.path, l10n_paths[mozpack.path.basedir(e.path, bases)][e.name]) for e in entries if isinstance(e, ManifestEntryWithRelPath)) for path in NON_CHROME: for p, f in l10n_finder.find(path): paths[p] = p # Create a new package, with non localized bits coming from the original # package, and localized bits coming from the langpack. packager = SimplePackager(formatter) for p, f in finder: if is_manifest(p): # Remove localized manifest entries. for e in [e for e in f if e.localized]: f.remove(e) base = mozpack.path.basedir(p, paths.keys()) if base: # If the path is one that needs a locale replacement, use the # corresponding file from the langpack. subpath = mozpack.path.relpath(p, base) path = mozpack.path.normpath(mozpack.path.join(paths[base], subpath)) files = [f for p, f in l10n_finder.find(path)] if len(files) == 0 and base in NON_CHROME: path = path.replace(locale, l10n_locale) files = [f for p, f in l10n_finder.find(path)] if len(files) == 0: if not base in NON_CHROME: errors.error("Missing file: %s" % os.path.join(l10n, path)) else: packager.add(path, files[0]) else: packager.add(p, f) # Add localized manifest entries from the langpack. l10n_manifests = [] for base in set(e.base for e in l10n_entries): m = ManifestFile(base) for e in l10n_entries: if e.base == base: m.add(e) path = mozpack.path.join(base, 'chrome.%s.manifest' % l10n_locale) l10n_manifests.append((path, m)) bases = packager.get_bases() for path, m in l10n_manifests: base = mozpack.path.basedir(path, bases) packager.add(path, m) # Add a "manifest $path" entry in the top manifest under that base. m = ManifestFile(base) m.add(Manifest(base, mozpack.path.relpath(path, base))) packager.add(mozpack.path.join(base, 'chrome.manifest'), m) packager.close() # Add any remaining non chrome files. for base in NON_CHROME: for p, f in l10n_finder.find(base): if not formatter.contains(p): formatter.add(p, f) # Transplant jar preloading information. for path, log in finder.jarlogs.iteritems(): assert isinstance(copier[path], Jarrer) copier[path].preload([l.replace(locale, l10n_locale) for l in log]) copier.copy(source) generate_precomplete(source)
def test_simple_packager(self): class GeneratedFileWithPath(GeneratedFile): def __init__(self, path, content): GeneratedFile.__init__(self, content) self.path = path formatter = MockFormatter() packager = SimplePackager(formatter) curdir = os.path.abspath(os.curdir) file = GeneratedFileWithPath(os.path.join(curdir, 'foo', 'bar.manifest'), 'resource bar bar/\ncontent bar bar/') with errors.context('manifest', 1): packager.add('foo/bar.manifest', file) file = GeneratedFileWithPath(os.path.join(curdir, 'foo', 'baz.manifest'), 'resource baz baz/') with errors.context('manifest', 2): packager.add('bar/baz.manifest', file) with errors.context('manifest', 3): packager.add('qux/qux.manifest', GeneratedFile('resource qux qux/')) bar_xpt = GeneratedFile('bar.xpt') qux_xpt = GeneratedFile('qux.xpt') foo_html = GeneratedFile('foo_html') bar_html = GeneratedFile('bar_html') with errors.context('manifest', 4): packager.add('foo/bar.xpt', bar_xpt) with errors.context('manifest', 5): packager.add('foo/bar/foo.html', foo_html) packager.add('foo/bar/bar.html', bar_html) file = GeneratedFileWithPath(os.path.join(curdir, 'foo.manifest'), ''.join([ 'manifest foo/bar.manifest\n', 'manifest bar/baz.manifest\n', ])) with errors.context('manifest', 6): packager.add('foo.manifest', file) with errors.context('manifest', 7): packager.add('foo/qux.xpt', qux_xpt) self.assertEqual(formatter.log, []) with errors.context('dummy', 1): packager.close() self.maxDiff = None self.assertEqual(formatter.log, [ (('dummy', 1), 'add_base', 'qux'), ((os.path.join(curdir, 'foo', 'bar.manifest'), 1), 'add_manifest', ManifestResource('foo', 'bar', 'bar/')), ((os.path.join(curdir, 'foo', 'bar.manifest'), 2), 'add_manifest', ManifestContent('foo', 'bar', 'bar/')), (('bar/baz.manifest', 1), 'add_manifest', ManifestResource('bar', 'baz', 'baz/')), (('qux/qux.manifest', 1), 'add_manifest', ManifestResource('qux', 'qux', 'qux/')), (('manifest', 4), 'add_interfaces', 'foo/bar.xpt', bar_xpt), (('manifest', 7), 'add_interfaces', 'foo/qux.xpt', qux_xpt), (('manifest', 5), 'add', 'foo/bar/foo.html', foo_html), (('manifest', 5), 'add', 'foo/bar/bar.html', bar_html), ]) self.assertEqual(packager.get_bases(), set(['', 'qux']))
def test_simple_packager(self): class GeneratedFileWithPath(GeneratedFile): def __init__(self, path, content): GeneratedFile.__init__(self, content) self.path = path formatter = MockFormatter() packager = SimplePackager(formatter) curdir = os.path.abspath(os.curdir) file = GeneratedFileWithPath( os.path.join(curdir, 'foo', 'bar.manifest'), 'resource bar bar/\ncontent bar bar/') with errors.context('manifest', 1): packager.add('foo/bar.manifest', file) file = GeneratedFileWithPath( os.path.join(curdir, 'foo', 'baz.manifest'), 'resource baz baz/') with errors.context('manifest', 2): packager.add('bar/baz.manifest', file) with errors.context('manifest', 3): packager.add('qux/qux.manifest', GeneratedFile('resource qux qux/')) bar_xpt = GeneratedFile('bar.xpt') qux_xpt = GeneratedFile('qux.xpt') foo_html = GeneratedFile('foo_html') bar_html = GeneratedFile('bar_html') with errors.context('manifest', 4): packager.add('foo/bar.xpt', bar_xpt) with errors.context('manifest', 5): packager.add('foo/bar/foo.html', foo_html) packager.add('foo/bar/bar.html', bar_html) file = GeneratedFileWithPath( os.path.join(curdir, 'foo.manifest'), ''.join([ 'manifest foo/bar.manifest\n', 'manifest bar/baz.manifest\n', ])) with errors.context('manifest', 6): packager.add('foo.manifest', file) with errors.context('manifest', 7): packager.add('foo/qux.xpt', qux_xpt) self.assertEqual(formatter.log, []) with errors.context('dummy', 1): packager.close() self.maxDiff = None self.assertEqual(formatter.log, [ (('dummy', 1), 'add_base', 'qux'), ((os.path.join(curdir, 'foo', 'bar.manifest'), 1), 'add_manifest', ManifestResource('foo', 'bar', 'bar/')), ((os.path.join(curdir, 'foo', 'bar.manifest'), 2), 'add_manifest', ManifestContent('foo', 'bar', 'bar/')), (('bar/baz.manifest', 1), 'add_manifest', ManifestResource('bar', 'baz', 'baz/')), (('qux/qux.manifest', 1), 'add_manifest', ManifestResource('qux', 'qux', 'qux/')), (('manifest', 4), 'add_interfaces', 'foo/bar.xpt', bar_xpt), (('manifest', 7), 'add_interfaces', 'foo/qux.xpt', qux_xpt), (('manifest', 5), 'add', 'foo/bar/foo.html', foo_html), (('manifest', 5), 'add', 'foo/bar/bar.html', bar_html), ]) self.assertEqual(packager.get_bases(), set(['', 'qux']))
def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()): app = LocaleManifestFinder(app_finder) l10n = LocaleManifestFinder(l10n_finder) # The code further below assumes there's only one locale replaced with # another one. if len(app.locales) > 1 or len(l10n.locales) > 1: errors.fatal("Multiple locales aren't supported") locale = app.locales[0] l10n_locale = l10n.locales[0] # For each base directory, store what path a locale chrome package name # corresponds to. # e.g., for the following entry under app/chrome: # locale foo en-US path/to/files # keep track that the locale path for foo in app is # app/chrome/path/to/files. l10n_paths = {} for e in l10n.entries: if isinstance(e, ManifestChrome): base = mozpack.path.basedir(e.path, app.bases) l10n_paths.setdefault(base, {}) l10n_paths[base][e.name] = e.path # For chrome and non chrome files or directories, store what langpack path # corresponds to a package path. paths = dict((e.path, l10n_paths[mozpack.path.basedir(e.path, app.bases)][e.name]) for e in app.entries if isinstance(e, ManifestEntryWithRelPath)) for pattern in non_chrome: for base in app.bases: path = mozpack.path.join(base, pattern) left = set(p for p, f in app_finder.find(path)) right = set(p for p, f in l10n_finder.find(path)) for p in right: paths[p] = p for p in left - right: paths[p] = None # Create a new package, with non localized bits coming from the original # package, and localized bits coming from the langpack. packager = SimplePackager(formatter) for p, f in app_finder: if is_manifest(p): # Remove localized manifest entries. for e in [e for e in f if e.localized]: f.remove(e) # If the path is one that needs a locale replacement, use the # corresponding file from the langpack. path = None if p in paths: path = paths[p] if not path: continue else: base = mozpack.path.basedir(p, paths.keys()) if base: subpath = mozpack.path.relpath(p, base) path = mozpack.path.normpath(mozpack.path.join(paths[base], subpath)) if path: files = [f for p, f in l10n_finder.find(path)] if not len(files): if base not in non_chrome: errors.error("Missing file: %s" % os.path.join(l10n_finder.base, path)) else: packager.add(path, files[0]) else: packager.add(p, f) # Add localized manifest entries from the langpack. l10n_manifests = [] for base in set(e.base for e in l10n.entries): m = ManifestFile(base, [e for e in l10n.entries if e.base == base]) path = mozpack.path.join(base, 'chrome.%s.manifest' % l10n_locale) l10n_manifests.append((path, m)) bases = packager.get_bases() for path, m in l10n_manifests: base = mozpack.path.basedir(path, bases) packager.add(path, m) # Add a "manifest $path" entry in the top manifest under that base. m = ManifestFile(base) m.add(Manifest(base, mozpack.path.relpath(path, base))) packager.add(mozpack.path.join(base, 'chrome.manifest'), m) packager.close() # Add any remaining non chrome files. for pattern in non_chrome: for base in bases: for p, f in l10n_finder.find(mozpack.path.join(base, pattern)): if not formatter.contains(p): formatter.add(p, f) # Transplant jar preloading information. for path, log in app_finder.jarlogs.iteritems(): assert isinstance(copier[path], Jarrer) copier[path].preload([l.replace(locale, l10n_locale) for l in log])