def is_resource(base, path): registry = FileRegistry() f = OmniJarFormatter( registry, "omni.foo", non_resources=[ "defaults/messenger/mailViews.dat", "defaults/foo/*", "*/dummy", ], ) f.add_base("") f.add_base("app") f.add(mozpath.join(base, path), GeneratedFile(b"")) if f.copier.contains(mozpath.join(base, path)): return False self.assertTrue(f.copier.contains(mozpath.join(base, "omni.foo"))) self.assertTrue(f.copier[mozpath.join(base, "omni.foo")].contains(path)) return True
def test_permissions(self): """Ensure files without write permission can be deleted.""" with open(self.tmppath('dummy'), 'a'): pass p = self.tmppath('no_perms') with open(p, 'a'): pass # Make file and directory unwritable. Reminder: making a directory # unwritable prevents modifications (including deletes) from the list # of files in that directory. os.chmod(p, 0400) os.chmod(self.tmpdir, 0400) copier = FileCopier() copier.add('dummy', GeneratedFile('content')) result = copier.copy(self.tmpdir) self.assertEqual(result.removed_files_count, 1) self.assertFalse(os.path.exists(p))
def test_no_remove(self): copier = FileCopier() copier.add('foo', GeneratedFile('foo')) with open(self.tmppath('bar'), 'a'): pass os.mkdir(self.tmppath('emptydir')) d = self.tmppath('populateddir') os.mkdir(d) with open(self.tmppath('populateddir/foo'), 'a'): pass result = copier.copy(self.tmpdir, remove_unaccounted=False) self.assertEqual(self.all_files(self.tmpdir), set(['foo', 'bar', 'populateddir/foo'])) self.assertEqual(result.removed_files, set()) self.assertEqual(result.removed_directories, set([self.tmppath('emptydir')]))
def test_file_copier(self): copier = FileCopier() copier.add('foo/bar', GeneratedFile('foobar')) copier.add('foo/qux', GeneratedFile('fooqux')) copier.add('foo/deep/nested/directory/file', GeneratedFile('fooz')) copier.add('bar', GeneratedFile('bar')) copier.add('qux/foo', GeneratedFile('quxfoo')) copier.add('qux/bar', GeneratedFile('')) copier.copy(self.tmpdir) self.assertEqual(self.all_files(self.tmpdir), set(copier.paths())) self.assertEqual(self.all_dirs(self.tmpdir), set(['foo/deep/nested/directory', 'qux'])) copier.remove('foo') copier.add('test', GeneratedFile('test')) copier.copy(self.tmpdir) self.assertEqual(self.all_files(self.tmpdir), set(copier.paths())) self.assertEqual(self.all_dirs(self.tmpdir), set(['qux']))
def test_jarrer(self): copier = Jarrer() copier.add("foo/bar", GeneratedFile(b"foobar")) copier.add("foo/qux", GeneratedFile(b"fooqux")) copier.add("foo/deep/nested/directory/file", GeneratedFile(b"fooz")) copier.add("bar", GeneratedFile(b"bar")) copier.add("qux/foo", GeneratedFile(b"quxfoo")) copier.add("qux/bar", GeneratedFile(b"")) dest = MockDest() copier.copy(dest) self.check_jar(dest, copier) copier.remove("foo") copier.add("test", GeneratedFile(b"test")) copier.copy(dest) self.check_jar(dest, copier) copier.remove("test") copier.add("test", GeneratedFile(b"replaced-content")) copier.copy(dest) self.check_jar(dest, copier) copier.copy(dest) self.check_jar(dest, copier) preloaded = ["qux/bar", "bar"] copier.preload(preloaded) copier.copy(dest) dest.seek(0) jar = JarReader(fileobj=dest) self.assertEqual( [f.filename for f in jar], preloaded + [p for p in copier.paths() if p not in preloaded], ) self.assertEqual(jar.last_preloaded, preloaded[-1])
def test_no_remove(self): copier = FileCopier() copier.add("foo", GeneratedFile(b"foo")) with open(self.tmppath("bar"), "a"): pass os.mkdir(self.tmppath("emptydir")) d = self.tmppath("populateddir") os.mkdir(d) with open(self.tmppath("populateddir/foo"), "a"): pass result = copier.copy(self.tmpdir, remove_unaccounted=False) self.assertEqual(self.all_files(self.tmpdir), set(["foo", "bar", "populateddir/foo"])) self.assertEqual(self.all_dirs(self.tmpdir), set(["populateddir"])) self.assertEqual(result.removed_files, set()) self.assertEqual(result.removed_directories, set([self.tmppath("emptydir")]))
def test_simple_manifest_parser(self): formatter = MockFormatter() foobar = GeneratedFile('foobar') foobaz = GeneratedFile('foobaz') fooqux = GeneratedFile('fooqux') foozot = GeneratedFile('foozot') finder = MockFinder({ 'bin/foo/bar': foobar, 'bin/foo/baz': foobaz, 'bin/foo/qux': fooqux, 'bin/foo/zot': foozot, 'bin/foo/chrome.manifest': GeneratedFile('resource foo foo/'), 'bin/chrome.manifest': GeneratedFile('manifest foo/chrome.manifest'), }) parser = SimpleManifestSink(finder, formatter) component0 = Component('component0') component1 = Component('component1') component2 = Component('component2', destdir='destdir') parser.add(component0, 'bin/foo/b*') parser.add(component1, 'bin/foo/qux') parser.add(component1, 'bin/foo/chrome.manifest') parser.add(component2, 'bin/foo/zot') self.assertRaises(ErrorMessage, parser.add, 'component1', 'bin/bar') self.assertEqual(formatter.log, []) parser.close() self.assertEqual(formatter.log, [ (None, 'add_base', '', False), (('foo/chrome.manifest', 1), 'add_manifest', ManifestResource('foo', 'foo', 'foo/')), (None, 'add', 'foo/bar', foobar), (None, 'add', 'foo/baz', foobaz), (None, 'add', 'foo/qux', fooqux), (None, 'add', 'destdir/foo/zot', foozot), ]) self.assertEqual(finder.log, [ 'bin/foo/b*', 'bin/foo/qux', 'bin/foo/chrome.manifest', 'bin/foo/zot', 'bin/bar', 'bin/chrome.manifest' ])
def add(self, path): self.registry.add(path, GeneratedFile(path))
def test_remove_unaccounted_directory_symlinks(self): """Directory symlinks in destination that are not in the way are deleted according to remove_unaccounted and remove_all_directory_symlinks. """ if not self.symlink_supported: return dest = self.tmppath("dest") copier = FileCopier() copier.add("foo/bar/baz", GeneratedFile(b"foobarbaz")) os.makedirs(self.tmppath("dest/foo")) dummy = self.tmppath("dummy") os.mkdir(dummy) os.mkdir(self.tmppath("dest/zot")) link = self.tmppath("dest/zot/zap") os.symlink(dummy, link) # If not remove_unaccounted but remove_empty_directories, then # the symlinked directory remains (as does its containing # directory). result = copier.copy( dest, remove_unaccounted=False, remove_empty_directories=True, remove_all_directory_symlinks=False, ) st = os.lstat(link) self.assertTrue(stat.S_ISLNK(st.st_mode)) self.assertFalse(stat.S_ISDIR(st.st_mode)) self.assertEqual(self.all_files(dest), set(copier.paths())) self.assertEqual(self.all_dirs(dest), set(["foo/bar"])) self.assertEqual(result.removed_directories, set()) self.assertEqual(len(result.updated_files), 1) # If remove_unaccounted but not remove_empty_directories, then # only the symlinked directory is removed. result = copier.copy( dest, remove_unaccounted=True, remove_empty_directories=False, remove_all_directory_symlinks=False, ) st = os.lstat(self.tmppath("dest/zot")) self.assertFalse(stat.S_ISLNK(st.st_mode)) self.assertTrue(stat.S_ISDIR(st.st_mode)) self.assertEqual(result.removed_files, set([link])) self.assertEqual(result.removed_directories, set()) self.assertEqual(self.all_files(dest), set(copier.paths())) self.assertEqual(self.all_dirs(dest), set(["foo/bar", "zot"])) # If remove_unaccounted and remove_empty_directories, then # both the symlink and its containing directory are removed. link = self.tmppath("dest/zot/zap") os.symlink(dummy, link) result = copier.copy( dest, remove_unaccounted=True, remove_empty_directories=True, remove_all_directory_symlinks=False, ) self.assertEqual(result.removed_files, set([link])) self.assertEqual(result.removed_directories, set([self.tmppath("dest/zot")])) self.assertEqual(self.all_files(dest), set(copier.paths())) self.assertEqual(self.all_dirs(dest), set(["foo/bar"]))
def create_registry(self): registry = FileRegistry() registry.add("foo/bar", GeneratedFile(b"foo/bar")) registry.add("baz/qux", GeneratedFile(b"baz/qux")) return FileRegistrySubtree("base/root", registry)
def test_l10n_repack(self): foo = GeneratedFile(b"foo") foobar = GeneratedFile(b"foobar") qux = GeneratedFile(b"qux") bar = GeneratedFile(b"bar") baz = GeneratedFile(b"baz") dict_aa = GeneratedFile(b"dict_aa") dict_bb = GeneratedFile(b"dict_bb") dict_cc = GeneratedFile(b"dict_cc") barbaz = GeneratedFile(b"barbaz") lst = GeneratedFile(b"foo\nbar") app_finder = MockFinder({ "bar/foo": foo, "chrome/foo/foobar": foobar, "chrome/qux/qux.properties": qux, "chrome/qux/baz/baz.properties": baz, "chrome/chrome.manifest": ManifestFile( "chrome", [ ManifestContent("chrome", "foo", "foo/"), ManifestLocale("chrome", "qux", "en-US", "qux/"), ], ), "chrome.manifest": ManifestFile("", [Manifest("", "chrome/chrome.manifest")]), "dict/aa": dict_aa, "app/chrome/bar/barbaz.dtd": barbaz, "app/chrome/chrome.manifest": ManifestFile( "app/chrome", [ManifestLocale("app/chrome", "bar", "en-US", "bar/")]), "app/chrome.manifest": ManifestFile("app", [Manifest("app", "chrome/chrome.manifest")]), "app/dict/bb": dict_bb, "app/dict/cc": dict_cc, "app/chrome/bar/search/foo.xml": foo, "app/chrome/bar/search/bar.xml": bar, "app/chrome/bar/search/lst.txt": lst, }) app_finder.jarlogs = {} app_finder.base = "app" foo_l10n = GeneratedFile(b"foo_l10n") qux_l10n = GeneratedFile(b"qux_l10n") baz_l10n = GeneratedFile(b"baz_l10n") barbaz_l10n = GeneratedFile(b"barbaz_l10n") lst_l10n = GeneratedFile(b"foo\nqux") l10n_finder = MockFinder({ "chrome/qux-l10n/qux.properties": qux_l10n, "chrome/qux-l10n/baz/baz.properties": baz_l10n, "chrome/chrome.manifest": ManifestFile( "chrome", [ ManifestLocale("chrome", "qux", "x-test", "qux-l10n/"), ], ), "chrome.manifest": ManifestFile("", [Manifest("", "chrome/chrome.manifest")]), "dict/bb": dict_bb, "dict/cc": dict_cc, "app/chrome/bar-l10n/barbaz.dtd": barbaz_l10n, "app/chrome/chrome.manifest": ManifestFile( "app/chrome", [ManifestLocale("app/chrome", "bar", "x-test", "bar-l10n/")], ), "app/chrome.manifest": ManifestFile("app", [Manifest("app", "chrome/chrome.manifest")]), "app/dict/aa": dict_aa, "app/chrome/bar-l10n/search/foo.xml": foo_l10n, "app/chrome/bar-l10n/search/qux.xml": qux_l10n, "app/chrome/bar-l10n/search/lst.txt": lst_l10n, }) l10n_finder.base = "l10n" copier = FileRegistry() formatter = FlatFormatter(copier) l10n._repack( app_finder, l10n_finder, copier, formatter, ["dict", "chrome/**/search/*.xml"], ) self.maxDiff = None repacked = { "bar/foo": foo, "chrome/foo/foobar": foobar, "chrome/qux-l10n/qux.properties": qux_l10n, "chrome/qux-l10n/baz/baz.properties": baz_l10n, "chrome/chrome.manifest": ManifestFile( "chrome", [ ManifestContent("chrome", "foo", "foo/"), ManifestLocale("chrome", "qux", "x-test", "qux-l10n/"), ], ), "chrome.manifest": ManifestFile("", [Manifest("", "chrome/chrome.manifest")]), "dict/bb": dict_bb, "dict/cc": dict_cc, "app/chrome/bar-l10n/barbaz.dtd": barbaz_l10n, "app/chrome/chrome.manifest": ManifestFile( "app/chrome", [ManifestLocale("app/chrome", "bar", "x-test", "bar-l10n/")], ), "app/chrome.manifest": ManifestFile("app", [Manifest("app", "chrome/chrome.manifest")]), "app/dict/aa": dict_aa, "app/chrome/bar-l10n/search/foo.xml": foo_l10n, "app/chrome/bar-l10n/search/qux.xml": qux_l10n, "app/chrome/bar-l10n/search/lst.txt": lst_l10n, } self.assertEqual( dict((p, f.open().read()) for p, f in copier), dict((p, f.open().read()) for p, f in six.iteritems(repacked)), )
def test_omnijar_formatter(self): registry = FileRegistry() formatter = OmniJarFormatter(registry, 'omni.foo') formatter.add_base('app') formatter.add('chrome/f/oo/bar', GeneratedFile('foobar')) formatter.add('chrome/f/oo/baz', GeneratedFile('foobaz')) formatter.add('chrome/f/oo/qux', GeneratedFile('fooqux')) formatter.add_manifest(ManifestContent('chrome/f/oo', 'bar', 'bar')) formatter.add_manifest(ManifestContent('chrome/f/oo', 'qux', 'qux')) self.assertEqual(registry.paths(), ['omni.foo']) self.assertEqual(registry['omni.foo'].paths(), [ 'chrome/f/oo/bar', 'chrome/f/oo/baz', 'chrome/f/oo/qux', 'chrome.manifest', 'chrome/chrome.manifest', 'chrome/f/f.manifest', 'chrome/f/oo/oo.manifest', ]) self.assertEqual(registry['omni.foo']['chrome.manifest'].open().read(), 'manifest chrome/chrome.manifest\n') self.assertEqual( registry['omni.foo']['chrome/chrome.manifest'].open().read(), 'manifest f/f.manifest\n') self.assertEqual( registry['omni.foo']['chrome/f/f.manifest'].open().read(), 'manifest oo/oo.manifest\n') self.assertEqual( registry['omni.foo']['chrome/f/oo/oo.manifest'].open().read(), ''.join([ 'content bar bar\n', 'content qux qux\n', ])) self.assertTrue(formatter.contains('chrome/f/oo/bar')) self.assertFalse(formatter.contains('chrome/foo/bar')) formatter.add_interfaces('components/foo.xpt', foo_xpt) formatter.add_interfaces('components/bar.xpt', bar_xpt) self.assertEqual(registry['omni.foo'].paths(), [ 'chrome/f/oo/bar', 'chrome/f/oo/baz', 'chrome/f/oo/qux', 'chrome.manifest', 'chrome/chrome.manifest', 'chrome/f/f.manifest', 'chrome/f/oo/oo.manifest', 'components/components.manifest', 'components/interfaces.xpt', ]) self.assertEqual( registry['omni.foo']['chrome.manifest'].open().read(), ''.join([ 'manifest chrome/chrome.manifest\n', 'manifest components/components.manifest\n' ])) self.assertEqual( registry['omni.foo'] ['components/components.manifest'].open().read(), 'interfaces interfaces.xpt\n') registry['omni.foo']['components/interfaces.xpt'].copy( self.tmppath('interfaces.xpt')) linked = read_interfaces(self.tmppath('interfaces.xpt')) foo = read_interfaces(foo_xpt.open()) bar = read_interfaces(bar_xpt.open()) self.assertEqual(foo['foo'], linked['foo']) self.assertEqual(bar['bar'], linked['bar']) formatter.add('app/chrome/foo/baz', GeneratedFile('foobaz')) formatter.add_manifest(ManifestContent('app/chrome', 'content', 'foo/')) self.assertEqual(registry.paths(), ['omni.foo', 'app/omni.foo']) self.assertEqual(registry['app/omni.foo'].paths(), [ 'chrome/foo/baz', 'chrome.manifest', 'chrome/chrome.manifest', ]) self.assertEqual( registry['app/omni.foo']['chrome.manifest'].open().read(), 'manifest chrome/chrome.manifest\n') self.assertEqual( registry['app/omni.foo']['chrome/chrome.manifest'].open().read(), 'content content foo/\n') formatter.add_manifest(ManifestBinaryComponent('components', 'foo.so')) formatter.add('components/foo.so', GeneratedFile('foo')) self.assertEqual(registry.paths(), [ 'omni.foo', 'app/omni.foo', 'chrome.manifest', 'components/components.manifest', 'components/foo.so', ]) self.assertEqual(registry['chrome.manifest'].open().read(), 'manifest components/components.manifest\n') self.assertEqual( registry['components/components.manifest'].open().read(), 'binary-component foo.so\n') formatter.add_manifest( ManifestBinaryComponent('app/components', 'foo.so')) formatter.add('app/components/foo.so', GeneratedFile('foo')) self.assertEqual(registry.paths(), [ 'omni.foo', 'app/omni.foo', 'chrome.manifest', 'components/components.manifest', 'components/foo.so', 'app/chrome.manifest', 'app/components/components.manifest', 'app/components/foo.so', ]) self.assertEqual(registry['app/chrome.manifest'].open().read(), 'manifest components/components.manifest\n') self.assertEqual( registry['app/components/components.manifest'].open().read(), 'binary-component foo.so\n') formatter.add('app/foo', GeneratedFile('foo')) self.assertEqual(registry.paths(), [ 'omni.foo', 'app/omni.foo', 'chrome.manifest', 'components/components.manifest', 'components/foo.so', 'app/chrome.manifest', 'app/components/components.manifest', 'app/components/foo.so', 'app/foo' ])
def test_simple_packager(self): class GeneratedFileWithPath(GeneratedFile): def __init__(self, path, content): GeneratedFile.__init__(self, content) self.path = path formatter = MockFormatter() packager = SimplePackager(formatter) curdir = os.path.abspath(os.curdir) file = GeneratedFileWithPath(os.path.join(curdir, 'foo', 'bar.manifest'), 'resource bar bar/\ncontent bar bar/') with errors.context('manifest', 1): packager.add('foo/bar.manifest', file) file = GeneratedFileWithPath(os.path.join(curdir, 'foo', 'baz.manifest'), 'resource baz baz/') with errors.context('manifest', 2): packager.add('bar/baz.manifest', file) with errors.context('manifest', 3): packager.add('qux/qux.manifest', GeneratedFile('resource qux qux/')) bar_xpt = GeneratedFile('bar.xpt') qux_xpt = GeneratedFile('qux.xpt') foo_html = GeneratedFile('foo_html') bar_html = GeneratedFile('bar_html') with errors.context('manifest', 4): packager.add('foo/bar.xpt', bar_xpt) with errors.context('manifest', 5): packager.add('foo/bar/foo.html', foo_html) packager.add('foo/bar/bar.html', bar_html) file = GeneratedFileWithPath(os.path.join(curdir, 'foo.manifest'), ''.join([ 'manifest foo/bar.manifest\n', 'manifest bar/baz.manifest\n', ])) with errors.context('manifest', 6): packager.add('foo.manifest', file) with errors.context('manifest', 7): packager.add('foo/qux.xpt', qux_xpt) file = GeneratedFileWithPath(os.path.join(curdir, 'addon', 'chrome.manifest'), 'resource hoge hoge/') with errors.context('manifest', 8): packager.add('addon/chrome.manifest', file) install_rdf = GeneratedFile('<RDF></RDF>') with errors.context('manifest', 9): packager.add('addon/install.rdf', install_rdf) self.assertEqual(formatter.log, []) with errors.context('dummy', 1): packager.close() self.maxDiff = None # The formatter is expected to reorder the manifest entries so that # chrome entries appear before the others. self.assertEqual(formatter.log, [ (('dummy', 1), 'add_base', 'qux', False), (('dummy', 1), 'add_base', 'addon', True), ((os.path.join(curdir, 'foo', 'bar.manifest'), 2), 'add_manifest', ManifestContent('foo', 'bar', 'bar/')), ((os.path.join(curdir, 'foo', 'bar.manifest'), 1), 'add_manifest', ManifestResource('foo', 'bar', 'bar/')), (('bar/baz.manifest', 1), 'add_manifest', ManifestResource('bar', 'baz', 'baz/')), (('qux/qux.manifest', 1), 'add_manifest', ManifestResource('qux', 'qux', 'qux/')), (('manifest', 4), 'add_interfaces', 'foo/bar.xpt', bar_xpt), (('manifest', 7), 'add_interfaces', 'foo/qux.xpt', qux_xpt), ((os.path.join(curdir, 'addon', 'chrome.manifest'), 1), 'add_manifest', ManifestResource('addon', 'hoge', 'hoge/')), (('manifest', 5), 'add', 'foo/bar/foo.html', foo_html), (('manifest', 5), 'add', 'foo/bar/bar.html', bar_html), (('manifest', 9), 'add', 'addon/install.rdf', install_rdf), ]) self.assertEqual(packager.get_bases(), set(['', 'addon', 'qux'])) self.assertEqual(packager.get_bases(addons=False), set(['', 'qux']))
def test_simple_packager(self): class GeneratedFileWithPath(GeneratedFile): def __init__(self, path, content): GeneratedFile.__init__(self, content) self.path = path formatter = MockFormatter() packager = SimplePackager(formatter) curdir = os.path.abspath(os.curdir) file = GeneratedFileWithPath(os.path.join(curdir, 'foo', 'bar.manifest'), 'resource bar bar/\ncontent bar bar/') with errors.context('manifest', 1): packager.add('foo/bar.manifest', file) file = GeneratedFileWithPath(os.path.join(curdir, 'foo', 'baz.manifest'), 'resource baz baz/') with errors.context('manifest', 2): packager.add('bar/baz.manifest', file) with errors.context('manifest', 3): packager.add('qux/qux.manifest', GeneratedFile(''.join([ 'resource qux qux/\n', 'binary-component qux.so\n', ]))) bar_xpt = GeneratedFile('bar.xpt') qux_xpt = GeneratedFile('qux.xpt') foo_html = GeneratedFile('foo_html') bar_html = GeneratedFile('bar_html') with errors.context('manifest', 4): packager.add('foo/bar.xpt', bar_xpt) with errors.context('manifest', 5): packager.add('foo/bar/foo.html', foo_html) packager.add('foo/bar/bar.html', bar_html) file = GeneratedFileWithPath(os.path.join(curdir, 'foo.manifest'), ''.join([ 'manifest foo/bar.manifest\n', 'manifest bar/baz.manifest\n', ])) with errors.context('manifest', 6): packager.add('foo.manifest', file) with errors.context('manifest', 7): packager.add('foo/qux.xpt', qux_xpt) file = GeneratedFileWithPath(os.path.join(curdir, 'addon', 'chrome.manifest'), 'resource hoge hoge/') with errors.context('manifest', 8): packager.add('addon/chrome.manifest', file) install_rdf = GeneratedFile('<RDF></RDF>') with errors.context('manifest', 9): packager.add('addon/install.rdf', install_rdf) with errors.context('manifest', 10): packager.add('addon2/install.rdf', install_rdf) packager.add('addon2/chrome.manifest', GeneratedFile('binary-component addon2.so')) with errors.context('manifest', 11): packager.add('addon3/install.rdf', install_rdf) packager.add('addon3/chrome.manifest', GeneratedFile( 'manifest components/components.manifest')) packager.add('addon3/components/components.manifest', GeneratedFile('binary-component addon3.so')) with errors.context('manifest', 12): install_rdf_addon4 = GeneratedFile( '<RDF>\n<...>\n<em:unpack>true</em:unpack>\n<...>\n</RDF>') packager.add('addon4/install.rdf', install_rdf_addon4) with errors.context('manifest', 13): install_rdf_addon5 = GeneratedFile( '<RDF>\n<...>\n<em:unpack>false</em:unpack>\n<...>\n</RDF>') packager.add('addon5/install.rdf', install_rdf_addon5) with errors.context('manifest', 14): install_rdf_addon6 = GeneratedFile( '<RDF>\n<... em:unpack=true>\n<...>\n</RDF>') packager.add('addon6/install.rdf', install_rdf_addon6) with errors.context('manifest', 15): install_rdf_addon7 = GeneratedFile( '<RDF>\n<... em:unpack=false>\n<...>\n</RDF>') packager.add('addon7/install.rdf', install_rdf_addon7) with errors.context('manifest', 16): install_rdf_addon8 = GeneratedFile( '<RDF>\n<... em:unpack="true">\n<...>\n</RDF>') packager.add('addon8/install.rdf', install_rdf_addon8) with errors.context('manifest', 17): install_rdf_addon9 = GeneratedFile( '<RDF>\n<... em:unpack="false">\n<...>\n</RDF>') packager.add('addon9/install.rdf', install_rdf_addon9) with errors.context('manifest', 18): install_rdf_addon10 = GeneratedFile( '<RDF>\n<... em:unpack=\'true\'>\n<...>\n</RDF>') packager.add('addon10/install.rdf', install_rdf_addon10) with errors.context('manifest', 19): install_rdf_addon11 = GeneratedFile( '<RDF>\n<... em:unpack=\'false\'>\n<...>\n</RDF>') packager.add('addon11/install.rdf', install_rdf_addon11) we_manifest = GeneratedFile( '{"manifest_version": 2, "name": "Test WebExtension", "version": "1.0"}') # hybrid and hybrid2 are both bootstrapped extensions with # embedded webextensions, they differ in the order in which # the manifests are added to the packager. with errors.context('manifest', 20): packager.add('hybrid/install.rdf', install_rdf) with errors.context('manifest', 21): packager.add('hybrid/webextension/manifest.json', we_manifest) with errors.context('manifest', 22): packager.add('hybrid2/webextension/manifest.json', we_manifest) with errors.context('manifest', 23): packager.add('hybrid2/install.rdf', install_rdf) with errors.context('manifest', 24): packager.add('webextension/manifest.json', we_manifest) non_we_manifest = GeneratedFile('{"not a webextension": true}') with errors.context('manifest', 25): packager.add('nonwebextension/manifest.json', non_we_manifest) self.assertEqual(formatter.log, []) with errors.context('dummy', 1): packager.close() self.maxDiff = None # The formatter is expected to reorder the manifest entries so that # chrome entries appear before the others. self.assertEqual(formatter.log, [ (('dummy', 1), 'add_base', '', False), (('dummy', 1), 'add_base', 'addon', True), (('dummy', 1), 'add_base', 'addon10', 'unpacked'), (('dummy', 1), 'add_base', 'addon11', True), (('dummy', 1), 'add_base', 'addon2', 'unpacked'), (('dummy', 1), 'add_base', 'addon3', 'unpacked'), (('dummy', 1), 'add_base', 'addon4', 'unpacked'), (('dummy', 1), 'add_base', 'addon5', True), (('dummy', 1), 'add_base', 'addon6', 'unpacked'), (('dummy', 1), 'add_base', 'addon7', True), (('dummy', 1), 'add_base', 'addon8', 'unpacked'), (('dummy', 1), 'add_base', 'addon9', True), (('dummy', 1), 'add_base', 'hybrid', True), (('dummy', 1), 'add_base', 'hybrid2', True), (('dummy', 1), 'add_base', 'qux', False), (('dummy', 1), 'add_base', 'webextension', True), ((os.path.join(curdir, 'foo', 'bar.manifest'), 2), 'add_manifest', ManifestContent('foo', 'bar', 'bar/')), ((os.path.join(curdir, 'foo', 'bar.manifest'), 1), 'add_manifest', ManifestResource('foo', 'bar', 'bar/')), (('bar/baz.manifest', 1), 'add_manifest', ManifestResource('bar', 'baz', 'baz/')), (('qux/qux.manifest', 1), 'add_manifest', ManifestResource('qux', 'qux', 'qux/')), (('qux/qux.manifest', 2), 'add_manifest', ManifestBinaryComponent('qux', 'qux.so')), (('manifest', 4), 'add_interfaces', 'foo/bar.xpt', bar_xpt), (('manifest', 7), 'add_interfaces', 'foo/qux.xpt', qux_xpt), ((os.path.join(curdir, 'addon', 'chrome.manifest'), 1), 'add_manifest', ManifestResource('addon', 'hoge', 'hoge/')), (('addon2/chrome.manifest', 1), 'add_manifest', ManifestBinaryComponent('addon2', 'addon2.so')), (('addon3/components/components.manifest', 1), 'add_manifest', ManifestBinaryComponent('addon3/components', 'addon3.so')), (('manifest', 5), 'add', 'foo/bar/foo.html', foo_html), (('manifest', 5), 'add', 'foo/bar/bar.html', bar_html), (('manifest', 9), 'add', 'addon/install.rdf', install_rdf), (('manifest', 10), 'add', 'addon2/install.rdf', install_rdf), (('manifest', 11), 'add', 'addon3/install.rdf', install_rdf), (('manifest', 12), 'add', 'addon4/install.rdf', install_rdf_addon4), (('manifest', 13), 'add', 'addon5/install.rdf', install_rdf_addon5), (('manifest', 14), 'add', 'addon6/install.rdf', install_rdf_addon6), (('manifest', 15), 'add', 'addon7/install.rdf', install_rdf_addon7), (('manifest', 16), 'add', 'addon8/install.rdf', install_rdf_addon8), (('manifest', 17), 'add', 'addon9/install.rdf', install_rdf_addon9), (('manifest', 18), 'add', 'addon10/install.rdf', install_rdf_addon10), (('manifest', 19), 'add', 'addon11/install.rdf', install_rdf_addon11), (('manifest', 20), 'add', 'hybrid/install.rdf', install_rdf), (('manifest', 21), 'add', 'hybrid/webextension/manifest.json', we_manifest), (('manifest', 22), 'add', 'hybrid2/webextension/manifest.json', we_manifest), (('manifest', 23), 'add', 'hybrid2/install.rdf', install_rdf), (('manifest', 24), 'add', 'webextension/manifest.json', we_manifest), (('manifest', 25), 'add', 'nonwebextension/manifest.json', non_we_manifest), ]) self.assertEqual(packager.get_bases(), set(['', 'addon', 'addon2', 'addon3', 'addon4', 'addon5', 'addon6', 'addon7', 'addon8', 'addon9', 'addon10', 'addon11', 'qux', 'hybrid', 'hybrid2', 'webextension'])) self.assertEqual(packager.get_bases(addons=False), set(['', 'qux']))
def create_context_tar(topsrcdir, context_dir, out_path, prefix, args=None): """Create a context tarball. A directory ``context_dir`` containing a Dockerfile will be assembled into a gzipped tar file at ``out_path``. Files inside the archive will be prefixed by directory ``prefix``. We also scan the source Dockerfile for special syntax that influences context generation. If a line in the Dockerfile has the form ``# %include <path>``, the relative path specified on that line will be matched against files in the source repository and added to the context under the path ``topsrcdir/``. If an entry is a directory, we add all files under that directory. If a line in the Dockerfile has the form ``# %ARG <name>``, occurrences of the string ``$<name>`` in subsequent lines are replaced with the value found in the ``args`` argument. Exception: this doesn't apply to VOLUME definitions. Returns the SHA-256 hex digest of the created archive. """ archive_files = {} replace = [] for root, dirs, files in os.walk(context_dir): for f in files: source_path = os.path.join(root, f) rel = source_path[len(context_dir) + 1:] archive_path = os.path.join(prefix, rel) archive_files[archive_path] = source_path # Parse Dockerfile for special syntax of extra files to include. content = [] with open(os.path.join(context_dir, 'Dockerfile'), 'rb') as fh: for line in fh: if line.startswith('# %ARG'): p = line[len('# %ARG '):].strip() if not args or p not in args: raise Exception('missing argument: {}'.format(p)) replace.append( (re.compile(r'\${}\b'.format(p)), args[p].encode('ascii'))) continue for regexp, s in replace: line = re.sub(regexp, s, line) content.append(line) if not line.startswith('# %include'): continue p = line[len('# %include '):].strip() if os.path.isabs(p): raise Exception('extra include path cannot be absolute: %s' % p) fs_path = os.path.normpath(os.path.join(topsrcdir, p)) # Check for filesystem traversal exploits. if not fs_path.startswith(topsrcdir): raise Exception('extra include path outside topsrcdir: %s' % p) if not os.path.exists(fs_path): raise Exception('extra include path does not exist: %s' % p) if os.path.isdir(fs_path): for root, dirs, files in os.walk(fs_path): for f in files: source_path = os.path.join(root, f) rel = source_path[len(fs_path) + 1:] archive_path = os.path.join(prefix, 'topsrcdir', p, rel) archive_files[archive_path] = source_path else: archive_path = os.path.join(prefix, 'topsrcdir', p) archive_files[archive_path] = fs_path archive_files[os.path.join(prefix, 'Dockerfile')] = \ GeneratedFile(b''.join(content)) with open(out_path, 'wb') as fh: create_tar_gz_from_files(fh, archive_files, '%s.tar.gz' % prefix) h = hashlib.sha256() with open(out_path, 'rb') as fh: while True: data = fh.read(32768) if not data: break h.update(data) return h.hexdigest()
'bases': { # base_path: is_addon? 'app': False, 'addon0': True, }, 'manifests': [ ManifestContent('chrome/f', 'oo', 'oo/'), ManifestContent('chrome/f', 'bar', 'oo/bar/'), ManifestResource('chrome/f', 'foo', 'resource://bar/'), ManifestBinaryComponent('components', 'foo.so'), ManifestContent('app/chrome', 'content', 'foo/'), ManifestComponent('app/components', '{foo-id}', 'foo.js'), ManifestContent('addon0/chrome', 'content', 'foo/bar/'), ], 'chrome/f/oo/bar/baz': GeneratedFile('foobarbaz'), 'chrome/f/oo/baz': GeneratedFile('foobaz'), 'chrome/f/oo/qux': GeneratedFile('fooqux'), 'components/foo.so': GeneratedFile('foo.so'), 'components/foo.xpt': foo_xpt, 'components/bar.xpt': bar_xpt, 'foo': GeneratedFile('foo'), 'app/chrome/foo/foo': GeneratedFile('appfoo'), 'app/components/foo.js':
def stream_context_tar(topsrcdir, context_dir, out_file, image_name, args): """Like create_context_tar, but streams the tar file to the `out_file` file object.""" archive_files = {} replace = [] content = [] context_dir = os.path.join(topsrcdir, context_dir) for root, dirs, files in os.walk(context_dir): for f in files: source_path = os.path.join(root, f) archive_path = source_path[len(context_dir) + 1:] archive_files[archive_path] = source_path # Parse Dockerfile for special syntax of extra files to include. with io.open(os.path.join(context_dir, "Dockerfile"), "r") as fh: for line in fh: if line.startswith("# %ARG"): p = line[len("# %ARG "):].strip() if not args or p not in args: raise Exception("missing argument: {}".format(p)) replace.append((re.compile(r"\${}\b".format(p)), args[p])) continue for regexp, s in replace: line = re.sub(regexp, s, line) content.append(line) if not line.startswith("# %include"): continue p = line[len("# %include "):].strip() if os.path.isabs(p): raise Exception("extra include path cannot be absolute: %s" % p) fs_path = os.path.normpath(os.path.join(topsrcdir, p)) # Check for filesystem traversal exploits. if not fs_path.startswith(topsrcdir): raise Exception("extra include path outside topsrcdir: %s" % p) if not os.path.exists(fs_path): raise Exception("extra include path does not exist: %s" % p) if os.path.isdir(fs_path): for root, dirs, files in os.walk(fs_path): for f in files: source_path = os.path.join(root, f) rel = source_path[len(fs_path) + 1:] archive_path = os.path.join("topsrcdir", p, rel) archive_files[archive_path] = source_path else: archive_path = os.path.join("topsrcdir", p) archive_files[archive_path] = fs_path archive_files["Dockerfile"] = GeneratedFile(b"".join( six.ensure_binary(s) for s in content)) writer = HashingWriter(out_file) create_tar_gz_from_files(writer, archive_files, "{}.tar".format(image_name)) return writer.hexdigest()
'app': False, 'addon0': 'unpacked', 'addon1': True, }, 'manifests': [ ManifestContent('chrome/f', 'oo', 'oo/'), ManifestContent('chrome/f', 'bar', 'oo/bar/'), ManifestResource('chrome/f', 'foo', 'resource://bar/'), ManifestBinaryComponent('components', 'foo.so'), ManifestContent('app/chrome', 'content', 'foo/'), ManifestComponent('app/components', '{foo-id}', 'foo.js'), ManifestContent('addon0/chrome', 'content', 'foo/bar/'), ManifestContent('addon1/chrome', 'content', 'foo/bar/'), ], 'files': { 'chrome/f/oo/bar/baz': GeneratedFile('foobarbaz'), 'chrome/f/oo/baz': GeneratedFile('foobaz'), 'chrome/f/oo/qux': GeneratedFile('fooqux'), 'components/foo.so': GeneratedFile('foo.so'), 'components/foo.xpt': foo_xpt, 'components/bar.xpt': bar_xpt, 'foo': GeneratedFile('foo'), 'app/chrome/foo/foo': GeneratedFile('appfoo'), 'app/components/foo.js': GeneratedFile('foo.js'), 'addon0/chrome/foo/bar/baz': GeneratedFile('foobarbaz'), 'addon0/components/foo.xpt': foo2_xpt, 'addon0/components/bar.xpt': bar_xpt, 'addon1/chrome/foo/bar/baz': GeneratedFile('foobarbaz'), 'addon1/components/foo.xpt': foo2_xpt, 'addon1/components/bar.xpt': bar_xpt, },
def test_l10n_repack(self): foo = GeneratedFile('foo') foobar = GeneratedFile('foobar') qux = GeneratedFile('qux') bar = GeneratedFile('bar') baz = GeneratedFile('baz') dict_aa = GeneratedFile('dict_aa') dict_bb = GeneratedFile('dict_bb') dict_cc = GeneratedFile('dict_cc') barbaz = GeneratedFile('barbaz') lst = GeneratedFile('foo\nbar') app_finder = MockFinder({ 'bar/foo': foo, 'chrome/foo/foobar': foobar, 'chrome/qux/qux.properties': qux, 'chrome/qux/baz/baz.properties': baz, 'chrome/chrome.manifest': ManifestFile('chrome', [ ManifestContent('chrome', 'foo', 'foo/'), ManifestLocale('chrome', 'qux', 'en-US', 'qux/'), ]), 'chrome.manifest': ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]), 'dict/aa': dict_aa, 'app/chrome/bar/barbaz.dtd': barbaz, 'app/chrome/chrome.manifest': ManifestFile( 'app/chrome', [ManifestLocale('app/chrome', 'bar', 'en-US', 'bar/')]), 'app/chrome.manifest': ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]), 'app/dict/bb': dict_bb, 'app/dict/cc': dict_cc, 'app/chrome/bar/search/foo.xml': foo, 'app/chrome/bar/search/bar.xml': bar, 'app/chrome/bar/search/lst.txt': lst, }) app_finder.jarlogs = {} app_finder.base = 'app' foo_l10n = GeneratedFile('foo_l10n') qux_l10n = GeneratedFile('qux_l10n') baz_l10n = GeneratedFile('baz_l10n') barbaz_l10n = GeneratedFile('barbaz_l10n') lst_l10n = GeneratedFile('foo\nqux') l10n_finder = MockFinder({ 'chrome/qux-l10n/qux.properties': qux_l10n, 'chrome/qux-l10n/baz/baz.properties': baz_l10n, 'chrome/chrome.manifest': ManifestFile('chrome', [ ManifestLocale('chrome', 'qux', 'x-test', 'qux-l10n/'), ]), 'chrome.manifest': ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]), 'dict/bb': dict_bb, 'dict/cc': dict_cc, 'app/chrome/bar-l10n/barbaz.dtd': barbaz_l10n, 'app/chrome/chrome.manifest': ManifestFile( 'app/chrome', [ManifestLocale('app/chrome', 'bar', 'x-test', 'bar-l10n/')]), 'app/chrome.manifest': ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]), 'app/dict/aa': dict_aa, 'app/chrome/bar-l10n/search/foo.xml': foo_l10n, 'app/chrome/bar-l10n/search/qux.xml': qux_l10n, 'app/chrome/bar-l10n/search/lst.txt': lst_l10n, }) l10n_finder.base = 'l10n' copier = FileRegistry() formatter = FlatFormatter(copier) l10n._repack(app_finder, l10n_finder, copier, formatter, ['dict', 'chrome/**/search/*.xml']) self.maxDiff = None repacked = { 'bar/foo': foo, 'chrome/foo/foobar': foobar, 'chrome/qux-l10n/qux.properties': qux_l10n, 'chrome/qux-l10n/baz/baz.properties': baz_l10n, 'chrome/chrome.manifest': ManifestFile('chrome', [ ManifestContent('chrome', 'foo', 'foo/'), ManifestLocale('chrome', 'qux', 'x-test', 'qux-l10n/'), ]), 'chrome.manifest': ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]), 'dict/bb': dict_bb, 'dict/cc': dict_cc, 'app/chrome/bar-l10n/barbaz.dtd': barbaz_l10n, 'app/chrome/chrome.manifest': ManifestFile( 'app/chrome', [ManifestLocale('app/chrome', 'bar', 'x-test', 'bar-l10n/')]), 'app/chrome.manifest': ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]), 'app/dict/aa': dict_aa, 'app/chrome/bar-l10n/search/foo.xml': foo_l10n, 'app/chrome/bar-l10n/search/qux.xml': qux_l10n, 'app/chrome/bar-l10n/search/lst.txt': lst_l10n, } self.assertEqual( dict((p, f.open().read()) for p, f in copier), dict((p, f.open().read()) for p, f in repacked.iteritems()))
def test_simple_packager(self): class GeneratedFileWithPath(GeneratedFile): def __init__(self, path, content): GeneratedFile.__init__(self, content) self.path = path formatter = MockFormatter() packager = SimplePackager(formatter) curdir = os.path.abspath(os.curdir) file = GeneratedFileWithPath( os.path.join(curdir, 'foo', 'bar.manifest'), 'resource bar bar/\ncontent bar bar/') with errors.context('manifest', 1): packager.add('foo/bar.manifest', file) file = GeneratedFileWithPath( os.path.join(curdir, 'foo', 'baz.manifest'), 'resource baz baz/') with errors.context('manifest', 2): packager.add('bar/baz.manifest', file) with errors.context('manifest', 3): packager.add('qux/qux.manifest', GeneratedFile('resource qux qux/')) bar_xpt = GeneratedFile('bar.xpt') qux_xpt = GeneratedFile('qux.xpt') foo_html = GeneratedFile('foo_html') bar_html = GeneratedFile('bar_html') with errors.context('manifest', 4): packager.add('foo/bar.xpt', bar_xpt) with errors.context('manifest', 5): packager.add('foo/bar/foo.html', foo_html) packager.add('foo/bar/bar.html', bar_html) file = GeneratedFileWithPath( os.path.join(curdir, 'foo.manifest'), ''.join([ 'manifest foo/bar.manifest\n', 'manifest bar/baz.manifest\n', ])) with errors.context('manifest', 6): packager.add('foo.manifest', file) with errors.context('manifest', 7): packager.add('foo/qux.xpt', qux_xpt) self.assertEqual(formatter.log, []) with errors.context('dummy', 1): packager.close() self.maxDiff = None self.assertEqual(formatter.log, [ (('dummy', 1), 'add_base', 'qux'), ((os.path.join(curdir, 'foo', 'bar.manifest'), 1), 'add_manifest', ManifestResource('foo', 'bar', 'bar/')), ((os.path.join(curdir, 'foo', 'bar.manifest'), 2), 'add_manifest', ManifestContent('foo', 'bar', 'bar/')), (('bar/baz.manifest', 1), 'add_manifest', ManifestResource('bar', 'baz', 'baz/')), (('qux/qux.manifest', 1), 'add_manifest', ManifestResource('qux', 'qux', 'qux/')), (('manifest', 4), 'add_interfaces', 'foo/bar.xpt', bar_xpt), (('manifest', 7), 'add_interfaces', 'foo/qux.xpt', qux_xpt), (('manifest', 5), 'add', 'foo/bar/foo.html', foo_html), (('manifest', 5), 'add', 'foo/bar/bar.html', bar_html), ]) self.assertEqual(packager.get_bases(), set(['', 'qux']))
def test_simple_packager_manifest_consistency(self): formatter = MockFormatter() # bar/ is detected as an addon because of install.rdf, but top-level # includes a manifest inside bar/. packager = SimplePackager(formatter) packager.add('base.manifest', GeneratedFile( 'manifest foo/bar.manifest\n' 'manifest bar/baz.manifest\n' )) packager.add('foo/bar.manifest', GeneratedFile('resource bar bar')) packager.add('bar/baz.manifest', GeneratedFile('resource baz baz')) packager.add('bar/install.rdf', GeneratedFile('')) with self.assertRaises(ErrorMessage) as e: packager.close() self.assertEqual(e.exception.message, 'Error: "bar/baz.manifest" is included from "base.manifest", ' 'which is outside "bar"') # bar/ is detected as a separate base because of chrome.manifest that # is included nowhere, but top-level includes another manifest inside # bar/. packager = SimplePackager(formatter) packager.add('base.manifest', GeneratedFile( 'manifest foo/bar.manifest\n' 'manifest bar/baz.manifest\n' )) packager.add('foo/bar.manifest', GeneratedFile('resource bar bar')) packager.add('bar/baz.manifest', GeneratedFile('resource baz baz')) packager.add('bar/chrome.manifest', GeneratedFile('resource baz baz')) with self.assertRaises(ErrorMessage) as e: packager.close() self.assertEqual(e.exception.message, 'Error: "bar/baz.manifest" is included from "base.manifest", ' 'which is outside "bar"') # bar/ is detected as a separate base because of chrome.manifest that # is included nowhere, but chrome.manifest includes baz.manifest from # the same directory. This shouldn't error out. packager = SimplePackager(formatter) packager.add('base.manifest', GeneratedFile( 'manifest foo/bar.manifest\n' )) packager.add('foo/bar.manifest', GeneratedFile('resource bar bar')) packager.add('bar/baz.manifest', GeneratedFile('resource baz baz')) packager.add('bar/chrome.manifest', GeneratedFile('manifest baz.manifest')) packager.close()
"app/chrome/addons/addon2": True, }, "manifests": [ ManifestContent("chrome/f", "oo", "oo/"), ManifestContent("chrome/f", "bar", "oo/bar/"), ManifestResource("chrome/f", "foo", "resource://bar/"), ManifestBinaryComponent("components", "foo.so"), ManifestContent("app/chrome", "content", "foo/"), ManifestComponent("app/components", "{foo-id}", "foo.js"), ManifestContent("addon0/chrome", "addon0", "foo/bar/"), ManifestContent("addon1/chrome", "addon1", "foo/bar/"), ManifestContent("app/chrome/addons/addon2/chrome", "addon2", "foo/bar/"), ], "files": { "chrome/f/oo/bar/baz": GeneratedFile(b"foobarbaz"), "chrome/f/oo/baz": GeneratedFile(b"foobaz"), "chrome/f/oo/qux": GeneratedFile(b"fooqux"), "components/foo.so": GeneratedFile(b"foo.so"), "components/foo.xpt": foo_xpt, "components/bar.xpt": bar_xpt, "foo": GeneratedFile(b"foo"), "app/chrome/foo/foo": GeneratedFile(b"appfoo"), "app/components/foo.js": GeneratedFile(b"foo.js"), "addon0/chrome/foo/bar/baz": GeneratedFile(b"foobarbaz"), "addon0/components/foo.xpt": foo2_xpt, "addon0/components/bar.xpt": bar_xpt, "addon1/chrome/foo/bar/baz": GeneratedFile(b"foobarbaz"), "addon1/components/foo.xpt": foo2_xpt, "addon1/components/bar.xpt": bar_xpt, "app/chrome/addons/addon2/chrome/foo/bar/baz":
def stream_context_tar(topsrcdir, context_dir, out_file, prefix, args=None): """Like create_context_tar, but streams the tar file to the `out_file` file object.""" archive_files = {} replace = [] content = [] context_dir = os.path.join(topsrcdir, context_dir) for root, dirs, files in os.walk(context_dir): for f in files: source_path = os.path.join(root, f) rel = source_path[len(context_dir) + 1:] archive_path = os.path.join(prefix, rel) archive_files[archive_path] = source_path # Parse Dockerfile for special syntax of extra files to include. with open(os.path.join(context_dir, 'Dockerfile'), 'rb') as fh: for line in fh: if line.startswith('# %ARG'): p = line[len('# %ARG '):].strip() if not args or p not in args: raise Exception('missing argument: {}'.format(p)) replace.append((re.compile(r'\${}\b'.format(p)), args[p].encode('ascii'))) continue for regexp, s in replace: line = re.sub(regexp, s, line) content.append(line) if not line.startswith('# %include'): continue p = line[len('# %include '):].strip() if os.path.isabs(p): raise Exception('extra include path cannot be absolute: %s' % p) fs_path = os.path.normpath(os.path.join(topsrcdir, p)) # Check for filesystem traversal exploits. if not fs_path.startswith(topsrcdir): raise Exception('extra include path outside topsrcdir: %s' % p) if not os.path.exists(fs_path): raise Exception('extra include path does not exist: %s' % p) if os.path.isdir(fs_path): for root, dirs, files in os.walk(fs_path): for f in files: source_path = os.path.join(root, f) rel = source_path[len(fs_path) + 1:] archive_path = os.path.join(prefix, 'topsrcdir', p, rel) archive_files[archive_path] = source_path else: archive_path = os.path.join(prefix, 'topsrcdir', p) archive_files[archive_path] = fs_path archive_files[os.path.join(prefix, 'Dockerfile')] = \ GeneratedFile(b''.join(content)) writer = HashingWriter(out_file) create_tar_gz_from_files(writer, archive_files, '%s.tar.gz' % prefix) return writer.hexdigest()
def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()): app = LocaleManifestFinder(app_finder) l10n = LocaleManifestFinder(l10n_finder) # The code further below assumes there's only one locale replaced with # another one. if len(app.locales) > 1: errors.fatal("Multiple app locales aren't supported: " + ",".join(app.locales)) if len(l10n.locales) > 1: errors.fatal("Multiple l10n locales aren't supported: " + ",".join(l10n.locales)) locale = app.locales[0] l10n_locale = l10n.locales[0] # For each base directory, store what path a locale chrome package name # corresponds to. # e.g., for the following entry under app/chrome: # locale foo en-US path/to/files # keep track that the locale path for foo in app is # app/chrome/path/to/files. # As there may be multiple locale entries with the same base, but with # different flags, that tracking takes the flags into account when there # are some. Example: # locale foo en-US path/to/files/win os=Win # locale foo en-US path/to/files/mac os=Darwin def key(entry): if entry.flags: return '%s %s' % (entry.name, entry.flags) return entry.name l10n_paths = {} for e in l10n.entries: if isinstance(e, ManifestChrome): base = mozpath.basedir(e.path, app.bases) l10n_paths.setdefault(base, {}) l10n_paths[base][key(e)] = e.path # For chrome and non chrome files or directories, store what langpack path # corresponds to a package path. paths = {} for e in app.entries: if isinstance(e, ManifestEntryWithRelPath): base = mozpath.basedir(e.path, app.bases) if base not in l10n_paths: errors.fatal("Locale doesn't contain %s/" % base) # Allow errors to accumulate continue if key(e) not in l10n_paths[base]: errors.fatal("Locale doesn't have a manifest entry for '%s'" % e.name) # Allow errors to accumulate continue paths[e.path] = l10n_paths[base][key(e)] for pattern in non_chrome: for base in app.bases: path = mozpath.join(base, pattern) left = set(p for p, f in app_finder.find(path)) right = set(p for p, f in l10n_finder.find(path)) for p in right: paths[p] = p for p in left - right: paths[p] = None # Create a new package, with non localized bits coming from the original # package, and localized bits coming from the langpack. packager = SimplePackager(formatter) built_in_addons = None for p, f in app_finder: if is_manifest(p): # Remove localized manifest entries. for e in [e for e in f if e.localized]: f.remove(e) # If the path is one that needs a locale replacement, use the # corresponding file from the langpack. path = None if p in paths: path = paths[p] if not path: continue else: base = mozpath.basedir(p, paths.keys()) if base: subpath = mozpath.relpath(p, base) path = mozpath.normpath(mozpath.join(paths[base], subpath)) if path: files = [f for p, f in l10n_finder.find(path)] if not len(files): if base not in non_chrome: finderBase = "" if hasattr(l10n_finder, 'base'): finderBase = l10n_finder.base errors.error("Missing file: %s" % os.path.join(finderBase, path)) else: packager.add(path, files[0]) elif p.endswith('built_in_addons.json'): built_in_addons = (p, f) else: packager.add(p, f) # Add localized manifest entries from the langpack. l10n_manifests = [] for base in set(e.base for e in l10n.entries): m = ManifestFile(base, [e for e in l10n.entries if e.base == base]) path = mozpath.join(base, 'chrome.%s.manifest' % l10n_locale) l10n_manifests.append((path, m)) bases = packager.get_bases() for path, m in l10n_manifests: base = mozpath.basedir(path, bases) packager.add(path, m) # Add a "manifest $path" entry in the top manifest under that base. m = ManifestFile(base) m.add(Manifest(base, mozpath.relpath(path, base))) packager.add(mozpath.join(base, 'chrome.manifest'), m) packager.close() dictionaries = {} # Add any remaining non chrome files. for pattern in non_chrome: for base in bases: for p, f in l10n_finder.find(mozpath.join(base, pattern)): if not formatter.contains(p): if p.startswith('dictionaries/') and p.endswith('.dic'): base, ext = os.path.splitext(os.path.basename(p)) dictionaries[base] = p formatter.add(p, f) # Update the built-in add-ons manifest with the new list of dictionaries # from the langpack. if built_in_addons: data = json.load(built_in_addons[1].open()) data['dictionaries'] = dictionaries formatter.add(built_in_addons[0], GeneratedFile(json.dumps(data))) # Resources in `localization` directories are packaged from the source and then # if localized versions are present in the l10n dir, we package them as well # keeping the source dir resources as a runtime fallback. for p, f in l10n_finder.find('**/localization'): if not formatter.contains(p): formatter.add(p, f) # Transplant jar preloading information. for path, log in app_finder.jarlogs.iteritems(): assert isinstance(copier[path], Jarrer) copier[path].preload([l.replace(locale, l10n_locale) for l in log])
# Compiled typelib for the following IDL: # interface foo; # [scriptable, uuid(5f70da76-519c-4858-b71e-e3c92333e2d6)] # interface bar { # void bar(in foo f); # }; # We need to make this [scriptable] so it doesn't get deleted from the # typelib. We don't need to make the foo interfaces below [scriptable], # because they will be automatically included by virtue of being an # argument to a method of |bar|. bar_xpt = GeneratedFile( b"\x58\x50\x43\x4F\x4D\x0A\x54\x79\x70\x65\x4C\x69\x62\x0D\x0A\x1A" + b"\x01\x02\x00\x02\x00\x00\x00\x7B\x00\x00\x00\x24\x00\x00\x00\x5C" + b"\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x5F" + b"\x70\xDA\x76\x51\x9C\x48\x58\xB7\x1E\xE3\xC9\x23\x33\xE2\xD6\x00" + b"\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x0D\x00\x66\x6F\x6F\x00" + b"\x62\x61\x72\x00\x62\x61\x72\x00\x00\x00\x00\x01\x00\x00\x00\x00" + b"\x09\x01\x80\x92\x00\x01\x80\x06\x00\x00\x80") # Compiled typelib for the following IDL: # [uuid(3271bebc-927e-4bef-935e-44e0aaf3c1e5)] # interface foo { # void foo(); # }; foo_xpt = GeneratedFile( b"\x58\x50\x43\x4F\x4D\x0A\x54\x79\x70\x65\x4C\x69\x62\x0D\x0A\x1A" + b"\x01\x02\x00\x01\x00\x00\x00\x57\x00\x00\x00\x24\x00\x00\x00\x40" + b"\x80\x00\x00\x32\x71\xBE\xBC\x92\x7E\x4B\xEF\x93\x5E\x44\xE0\xAA" + b"\xF3\xC1\xE5\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x09\x00" +
def create_registry(self): registry = FileRegistry() registry.add('foo/bar', GeneratedFile('foo/bar')) registry.add('baz/qux', GeneratedFile('baz/qux')) return FileRegistrySubtree('base/root', registry)