def is_resource(base, path):
registry = FileRegistry()
f = OmniJarFormatter(
registry,
"omni.foo",
non_resources=[
"defaults/messenger/mailViews.dat",
"defaults/foo/*",
"*/dummy",
],
)
f.add_base("")
f.add_base("app")
f.add(mozpath.join(base, path), GeneratedFile(b""))
if f.copier.contains(mozpath.join(base, path)):
return False
self.assertTrue(f.copier.contains(mozpath.join(base, "omni.foo")))
self.assertTrue(f.copier[mozpath.join(base,
"omni.foo")].contains(path))
return True
def test_permissions(self):
"""Ensure files without write permission can be deleted."""
with open(self.tmppath('dummy'), 'a'):
pass
p = self.tmppath('no_perms')
with open(p, 'a'):
pass
# Make file and directory unwritable. Reminder: making a directory
# unwritable prevents modifications (including deletes) from the list
# of files in that directory.
os.chmod(p, 0400)
os.chmod(self.tmpdir, 0400)
copier = FileCopier()
copier.add('dummy', GeneratedFile('content'))
result = copier.copy(self.tmpdir)
self.assertEqual(result.removed_files_count, 1)
self.assertFalse(os.path.exists(p))
def test_no_remove(self):
copier = FileCopier()
copier.add('foo', GeneratedFile('foo'))
with open(self.tmppath('bar'), 'a'):
pass
os.mkdir(self.tmppath('emptydir'))
d = self.tmppath('populateddir')
os.mkdir(d)
with open(self.tmppath('populateddir/foo'), 'a'):
pass
result = copier.copy(self.tmpdir, remove_unaccounted=False)
self.assertEqual(self.all_files(self.tmpdir), set(['foo', 'bar',
'populateddir/foo']))
self.assertEqual(result.removed_files, set())
self.assertEqual(result.removed_directories,
set([self.tmppath('emptydir')]))
def test_file_copier(self):
copier = FileCopier()
copier.add('foo/bar', GeneratedFile('foobar'))
copier.add('foo/qux', GeneratedFile('fooqux'))
copier.add('foo/deep/nested/directory/file', GeneratedFile('fooz'))
copier.add('bar', GeneratedFile('bar'))
copier.add('qux/foo', GeneratedFile('quxfoo'))
copier.add('qux/bar', GeneratedFile(''))
copier.copy(self.tmpdir)
self.assertEqual(self.all_files(self.tmpdir), set(copier.paths()))
self.assertEqual(self.all_dirs(self.tmpdir),
set(['foo/deep/nested/directory', 'qux']))
copier.remove('foo')
copier.add('test', GeneratedFile('test'))
copier.copy(self.tmpdir)
self.assertEqual(self.all_files(self.tmpdir), set(copier.paths()))
self.assertEqual(self.all_dirs(self.tmpdir), set(['qux']))
def test_jarrer(self):
copier = Jarrer()
copier.add("foo/bar", GeneratedFile(b"foobar"))
copier.add("foo/qux", GeneratedFile(b"fooqux"))
copier.add("foo/deep/nested/directory/file", GeneratedFile(b"fooz"))
copier.add("bar", GeneratedFile(b"bar"))
copier.add("qux/foo", GeneratedFile(b"quxfoo"))
copier.add("qux/bar", GeneratedFile(b""))
dest = MockDest()
copier.copy(dest)
self.check_jar(dest, copier)
copier.remove("foo")
copier.add("test", GeneratedFile(b"test"))
copier.copy(dest)
self.check_jar(dest, copier)
copier.remove("test")
copier.add("test", GeneratedFile(b"replaced-content"))
copier.copy(dest)
self.check_jar(dest, copier)
copier.copy(dest)
self.check_jar(dest, copier)
preloaded = ["qux/bar", "bar"]
copier.preload(preloaded)
copier.copy(dest)
dest.seek(0)
jar = JarReader(fileobj=dest)
self.assertEqual(
[f.filename for f in jar],
preloaded + [p for p in copier.paths() if p not in preloaded],
)
self.assertEqual(jar.last_preloaded, preloaded[-1])
def test_no_remove(self):
copier = FileCopier()
copier.add("foo", GeneratedFile(b"foo"))
with open(self.tmppath("bar"), "a"):
pass
os.mkdir(self.tmppath("emptydir"))
d = self.tmppath("populateddir")
os.mkdir(d)
with open(self.tmppath("populateddir/foo"), "a"):
pass
result = copier.copy(self.tmpdir, remove_unaccounted=False)
self.assertEqual(self.all_files(self.tmpdir),
set(["foo", "bar", "populateddir/foo"]))
self.assertEqual(self.all_dirs(self.tmpdir), set(["populateddir"]))
self.assertEqual(result.removed_files, set())
self.assertEqual(result.removed_directories,
set([self.tmppath("emptydir")]))
def test_simple_manifest_parser(self):
formatter = MockFormatter()
foobar = GeneratedFile('foobar')
foobaz = GeneratedFile('foobaz')
fooqux = GeneratedFile('fooqux')
foozot = GeneratedFile('foozot')
finder = MockFinder({
'bin/foo/bar':
foobar,
'bin/foo/baz':
foobaz,
'bin/foo/qux':
fooqux,
'bin/foo/zot':
foozot,
'bin/foo/chrome.manifest':
GeneratedFile('resource foo foo/'),
'bin/chrome.manifest':
GeneratedFile('manifest foo/chrome.manifest'),
})
parser = SimpleManifestSink(finder, formatter)
component0 = Component('component0')
component1 = Component('component1')
component2 = Component('component2', destdir='destdir')
parser.add(component0, 'bin/foo/b*')
parser.add(component1, 'bin/foo/qux')
parser.add(component1, 'bin/foo/chrome.manifest')
parser.add(component2, 'bin/foo/zot')
self.assertRaises(ErrorMessage, parser.add, 'component1', 'bin/bar')
self.assertEqual(formatter.log, [])
parser.close()
self.assertEqual(formatter.log, [
(None, 'add_base', '', False),
(('foo/chrome.manifest', 1), 'add_manifest',
ManifestResource('foo', 'foo', 'foo/')),
(None, 'add', 'foo/bar', foobar),
(None, 'add', 'foo/baz', foobaz),
(None, 'add', 'foo/qux', fooqux),
(None, 'add', 'destdir/foo/zot', foozot),
])
self.assertEqual(finder.log, [
'bin/foo/b*', 'bin/foo/qux', 'bin/foo/chrome.manifest',
'bin/foo/zot', 'bin/bar', 'bin/chrome.manifest'
])
def add(self, path):
self.registry.add(path, GeneratedFile(path))
def test_remove_unaccounted_directory_symlinks(self):
"""Directory symlinks in destination that are not in the way are
deleted according to remove_unaccounted and
remove_all_directory_symlinks.
"""
if not self.symlink_supported:
return
dest = self.tmppath("dest")
copier = FileCopier()
copier.add("foo/bar/baz", GeneratedFile(b"foobarbaz"))
os.makedirs(self.tmppath("dest/foo"))
dummy = self.tmppath("dummy")
os.mkdir(dummy)
os.mkdir(self.tmppath("dest/zot"))
link = self.tmppath("dest/zot/zap")
os.symlink(dummy, link)
# If not remove_unaccounted but remove_empty_directories, then
# the symlinked directory remains (as does its containing
# directory).
result = copier.copy(
dest,
remove_unaccounted=False,
remove_empty_directories=True,
remove_all_directory_symlinks=False,
)
st = os.lstat(link)
self.assertTrue(stat.S_ISLNK(st.st_mode))
self.assertFalse(stat.S_ISDIR(st.st_mode))
self.assertEqual(self.all_files(dest), set(copier.paths()))
self.assertEqual(self.all_dirs(dest), set(["foo/bar"]))
self.assertEqual(result.removed_directories, set())
self.assertEqual(len(result.updated_files), 1)
# If remove_unaccounted but not remove_empty_directories, then
# only the symlinked directory is removed.
result = copier.copy(
dest,
remove_unaccounted=True,
remove_empty_directories=False,
remove_all_directory_symlinks=False,
)
st = os.lstat(self.tmppath("dest/zot"))
self.assertFalse(stat.S_ISLNK(st.st_mode))
self.assertTrue(stat.S_ISDIR(st.st_mode))
self.assertEqual(result.removed_files, set([link]))
self.assertEqual(result.removed_directories, set())
self.assertEqual(self.all_files(dest), set(copier.paths()))
self.assertEqual(self.all_dirs(dest), set(["foo/bar", "zot"]))
# If remove_unaccounted and remove_empty_directories, then
# both the symlink and its containing directory are removed.
link = self.tmppath("dest/zot/zap")
os.symlink(dummy, link)
result = copier.copy(
dest,
remove_unaccounted=True,
remove_empty_directories=True,
remove_all_directory_symlinks=False,
)
self.assertEqual(result.removed_files, set([link]))
self.assertEqual(result.removed_directories,
set([self.tmppath("dest/zot")]))
self.assertEqual(self.all_files(dest), set(copier.paths()))
self.assertEqual(self.all_dirs(dest), set(["foo/bar"]))
def create_registry(self):
registry = FileRegistry()
registry.add("foo/bar", GeneratedFile(b"foo/bar"))
registry.add("baz/qux", GeneratedFile(b"baz/qux"))
return FileRegistrySubtree("base/root", registry)
def test_l10n_repack(self):
foo = GeneratedFile(b"foo")
foobar = GeneratedFile(b"foobar")
qux = GeneratedFile(b"qux")
bar = GeneratedFile(b"bar")
baz = GeneratedFile(b"baz")
dict_aa = GeneratedFile(b"dict_aa")
dict_bb = GeneratedFile(b"dict_bb")
dict_cc = GeneratedFile(b"dict_cc")
barbaz = GeneratedFile(b"barbaz")
lst = GeneratedFile(b"foo\nbar")
app_finder = MockFinder({
"bar/foo":
foo,
"chrome/foo/foobar":
foobar,
"chrome/qux/qux.properties":
qux,
"chrome/qux/baz/baz.properties":
baz,
"chrome/chrome.manifest":
ManifestFile(
"chrome",
[
ManifestContent("chrome", "foo", "foo/"),
ManifestLocale("chrome", "qux", "en-US", "qux/"),
],
),
"chrome.manifest":
ManifestFile("", [Manifest("", "chrome/chrome.manifest")]),
"dict/aa":
dict_aa,
"app/chrome/bar/barbaz.dtd":
barbaz,
"app/chrome/chrome.manifest":
ManifestFile(
"app/chrome",
[ManifestLocale("app/chrome", "bar", "en-US", "bar/")]),
"app/chrome.manifest":
ManifestFile("app", [Manifest("app", "chrome/chrome.manifest")]),
"app/dict/bb":
dict_bb,
"app/dict/cc":
dict_cc,
"app/chrome/bar/search/foo.xml":
foo,
"app/chrome/bar/search/bar.xml":
bar,
"app/chrome/bar/search/lst.txt":
lst,
})
app_finder.jarlogs = {}
app_finder.base = "app"
foo_l10n = GeneratedFile(b"foo_l10n")
qux_l10n = GeneratedFile(b"qux_l10n")
baz_l10n = GeneratedFile(b"baz_l10n")
barbaz_l10n = GeneratedFile(b"barbaz_l10n")
lst_l10n = GeneratedFile(b"foo\nqux")
l10n_finder = MockFinder({
"chrome/qux-l10n/qux.properties":
qux_l10n,
"chrome/qux-l10n/baz/baz.properties":
baz_l10n,
"chrome/chrome.manifest":
ManifestFile(
"chrome",
[
ManifestLocale("chrome", "qux", "x-test", "qux-l10n/"),
],
),
"chrome.manifest":
ManifestFile("", [Manifest("", "chrome/chrome.manifest")]),
"dict/bb":
dict_bb,
"dict/cc":
dict_cc,
"app/chrome/bar-l10n/barbaz.dtd":
barbaz_l10n,
"app/chrome/chrome.manifest":
ManifestFile(
"app/chrome",
[ManifestLocale("app/chrome", "bar", "x-test", "bar-l10n/")],
),
"app/chrome.manifest":
ManifestFile("app", [Manifest("app", "chrome/chrome.manifest")]),
"app/dict/aa":
dict_aa,
"app/chrome/bar-l10n/search/foo.xml":
foo_l10n,
"app/chrome/bar-l10n/search/qux.xml":
qux_l10n,
"app/chrome/bar-l10n/search/lst.txt":
lst_l10n,
})
l10n_finder.base = "l10n"
copier = FileRegistry()
formatter = FlatFormatter(copier)
l10n._repack(
app_finder,
l10n_finder,
copier,
formatter,
["dict", "chrome/**/search/*.xml"],
)
self.maxDiff = None
repacked = {
"bar/foo":
foo,
"chrome/foo/foobar":
foobar,
"chrome/qux-l10n/qux.properties":
qux_l10n,
"chrome/qux-l10n/baz/baz.properties":
baz_l10n,
"chrome/chrome.manifest":
ManifestFile(
"chrome",
[
ManifestContent("chrome", "foo", "foo/"),
ManifestLocale("chrome", "qux", "x-test", "qux-l10n/"),
],
),
"chrome.manifest":
ManifestFile("", [Manifest("", "chrome/chrome.manifest")]),
"dict/bb":
dict_bb,
"dict/cc":
dict_cc,
"app/chrome/bar-l10n/barbaz.dtd":
barbaz_l10n,
"app/chrome/chrome.manifest":
ManifestFile(
"app/chrome",
[ManifestLocale("app/chrome", "bar", "x-test", "bar-l10n/")],
),
"app/chrome.manifest":
ManifestFile("app", [Manifest("app", "chrome/chrome.manifest")]),
"app/dict/aa":
dict_aa,
"app/chrome/bar-l10n/search/foo.xml":
foo_l10n,
"app/chrome/bar-l10n/search/qux.xml":
qux_l10n,
"app/chrome/bar-l10n/search/lst.txt":
lst_l10n,
}
self.assertEqual(
dict((p, f.open().read()) for p, f in copier),
dict((p, f.open().read()) for p, f in six.iteritems(repacked)),
)
def test_omnijar_formatter(self):
registry = FileRegistry()
formatter = OmniJarFormatter(registry, 'omni.foo')
formatter.add_base('app')
formatter.add('chrome/f/oo/bar', GeneratedFile('foobar'))
formatter.add('chrome/f/oo/baz', GeneratedFile('foobaz'))
formatter.add('chrome/f/oo/qux', GeneratedFile('fooqux'))
formatter.add_manifest(ManifestContent('chrome/f/oo', 'bar', 'bar'))
formatter.add_manifest(ManifestContent('chrome/f/oo', 'qux', 'qux'))
self.assertEqual(registry.paths(), ['omni.foo'])
self.assertEqual(registry['omni.foo'].paths(), [
'chrome/f/oo/bar',
'chrome/f/oo/baz',
'chrome/f/oo/qux',
'chrome.manifest',
'chrome/chrome.manifest',
'chrome/f/f.manifest',
'chrome/f/oo/oo.manifest',
])
self.assertEqual(registry['omni.foo']['chrome.manifest'].open().read(),
'manifest chrome/chrome.manifest\n')
self.assertEqual(
registry['omni.foo']['chrome/chrome.manifest'].open().read(),
'manifest f/f.manifest\n')
self.assertEqual(
registry['omni.foo']['chrome/f/f.manifest'].open().read(),
'manifest oo/oo.manifest\n')
self.assertEqual(
registry['omni.foo']['chrome/f/oo/oo.manifest'].open().read(),
''.join([
'content bar bar\n',
'content qux qux\n',
]))
self.assertTrue(formatter.contains('chrome/f/oo/bar'))
self.assertFalse(formatter.contains('chrome/foo/bar'))
formatter.add_interfaces('components/foo.xpt', foo_xpt)
formatter.add_interfaces('components/bar.xpt', bar_xpt)
self.assertEqual(registry['omni.foo'].paths(), [
'chrome/f/oo/bar',
'chrome/f/oo/baz',
'chrome/f/oo/qux',
'chrome.manifest',
'chrome/chrome.manifest',
'chrome/f/f.manifest',
'chrome/f/oo/oo.manifest',
'components/components.manifest',
'components/interfaces.xpt',
])
self.assertEqual(
registry['omni.foo']['chrome.manifest'].open().read(), ''.join([
'manifest chrome/chrome.manifest\n',
'manifest components/components.manifest\n'
]))
self.assertEqual(
registry['omni.foo']
['components/components.manifest'].open().read(),
'interfaces interfaces.xpt\n')
registry['omni.foo']['components/interfaces.xpt'].copy(
self.tmppath('interfaces.xpt'))
linked = read_interfaces(self.tmppath('interfaces.xpt'))
foo = read_interfaces(foo_xpt.open())
bar = read_interfaces(bar_xpt.open())
self.assertEqual(foo['foo'], linked['foo'])
self.assertEqual(bar['bar'], linked['bar'])
formatter.add('app/chrome/foo/baz', GeneratedFile('foobaz'))
formatter.add_manifest(ManifestContent('app/chrome', 'content',
'foo/'))
self.assertEqual(registry.paths(), ['omni.foo', 'app/omni.foo'])
self.assertEqual(registry['app/omni.foo'].paths(), [
'chrome/foo/baz',
'chrome.manifest',
'chrome/chrome.manifest',
])
self.assertEqual(
registry['app/omni.foo']['chrome.manifest'].open().read(),
'manifest chrome/chrome.manifest\n')
self.assertEqual(
registry['app/omni.foo']['chrome/chrome.manifest'].open().read(),
'content content foo/\n')
formatter.add_manifest(ManifestBinaryComponent('components', 'foo.so'))
formatter.add('components/foo.so', GeneratedFile('foo'))
self.assertEqual(registry.paths(), [
'omni.foo',
'app/omni.foo',
'chrome.manifest',
'components/components.manifest',
'components/foo.so',
])
self.assertEqual(registry['chrome.manifest'].open().read(),
'manifest components/components.manifest\n')
self.assertEqual(
registry['components/components.manifest'].open().read(),
'binary-component foo.so\n')
formatter.add_manifest(
ManifestBinaryComponent('app/components', 'foo.so'))
formatter.add('app/components/foo.so', GeneratedFile('foo'))
self.assertEqual(registry.paths(), [
'omni.foo',
'app/omni.foo',
'chrome.manifest',
'components/components.manifest',
'components/foo.so',
'app/chrome.manifest',
'app/components/components.manifest',
'app/components/foo.so',
])
self.assertEqual(registry['app/chrome.manifest'].open().read(),
'manifest components/components.manifest\n')
self.assertEqual(
registry['app/components/components.manifest'].open().read(),
'binary-component foo.so\n')
formatter.add('app/foo', GeneratedFile('foo'))
self.assertEqual(registry.paths(), [
'omni.foo', 'app/omni.foo', 'chrome.manifest',
'components/components.manifest', 'components/foo.so',
'app/chrome.manifest', 'app/components/components.manifest',
'app/components/foo.so', 'app/foo'
])
def test_simple_packager(self):
class GeneratedFileWithPath(GeneratedFile):
def __init__(self, path, content):
GeneratedFile.__init__(self, content)
self.path = path
formatter = MockFormatter()
packager = SimplePackager(formatter)
curdir = os.path.abspath(os.curdir)
file = GeneratedFileWithPath(os.path.join(curdir, 'foo',
'bar.manifest'),
'resource bar bar/\ncontent bar bar/')
with errors.context('manifest', 1):
packager.add('foo/bar.manifest', file)
file = GeneratedFileWithPath(os.path.join(curdir, 'foo',
'baz.manifest'),
'resource baz baz/')
with errors.context('manifest', 2):
packager.add('bar/baz.manifest', file)
with errors.context('manifest', 3):
packager.add('qux/qux.manifest',
GeneratedFile('resource qux qux/'))
bar_xpt = GeneratedFile('bar.xpt')
qux_xpt = GeneratedFile('qux.xpt')
foo_html = GeneratedFile('foo_html')
bar_html = GeneratedFile('bar_html')
with errors.context('manifest', 4):
packager.add('foo/bar.xpt', bar_xpt)
with errors.context('manifest', 5):
packager.add('foo/bar/foo.html', foo_html)
packager.add('foo/bar/bar.html', bar_html)
file = GeneratedFileWithPath(os.path.join(curdir, 'foo.manifest'),
''.join([
'manifest foo/bar.manifest\n',
'manifest bar/baz.manifest\n',
]))
with errors.context('manifest', 6):
packager.add('foo.manifest', file)
with errors.context('manifest', 7):
packager.add('foo/qux.xpt', qux_xpt)
file = GeneratedFileWithPath(os.path.join(curdir, 'addon',
'chrome.manifest'),
'resource hoge hoge/')
with errors.context('manifest', 8):
packager.add('addon/chrome.manifest', file)
install_rdf = GeneratedFile('<RDF></RDF>')
with errors.context('manifest', 9):
packager.add('addon/install.rdf', install_rdf)
self.assertEqual(formatter.log, [])
with errors.context('dummy', 1):
packager.close()
self.maxDiff = None
# The formatter is expected to reorder the manifest entries so that
# chrome entries appear before the others.
self.assertEqual(formatter.log, [
(('dummy', 1), 'add_base', 'qux', False),
(('dummy', 1), 'add_base', 'addon', True),
((os.path.join(curdir, 'foo', 'bar.manifest'), 2),
'add_manifest', ManifestContent('foo', 'bar', 'bar/')),
((os.path.join(curdir, 'foo', 'bar.manifest'), 1),
'add_manifest', ManifestResource('foo', 'bar', 'bar/')),
(('bar/baz.manifest', 1),
'add_manifest', ManifestResource('bar', 'baz', 'baz/')),
(('qux/qux.manifest', 1),
'add_manifest', ManifestResource('qux', 'qux', 'qux/')),
(('manifest', 4), 'add_interfaces', 'foo/bar.xpt', bar_xpt),
(('manifest', 7), 'add_interfaces', 'foo/qux.xpt', qux_xpt),
((os.path.join(curdir, 'addon', 'chrome.manifest'), 1),
'add_manifest', ManifestResource('addon', 'hoge', 'hoge/')),
(('manifest', 5), 'add', 'foo/bar/foo.html', foo_html),
(('manifest', 5), 'add', 'foo/bar/bar.html', bar_html),
(('manifest', 9), 'add', 'addon/install.rdf', install_rdf),
])
self.assertEqual(packager.get_bases(), set(['', 'addon', 'qux']))
self.assertEqual(packager.get_bases(addons=False), set(['', 'qux']))
def test_simple_packager(self):
class GeneratedFileWithPath(GeneratedFile):
def __init__(self, path, content):
GeneratedFile.__init__(self, content)
self.path = path
formatter = MockFormatter()
packager = SimplePackager(formatter)
curdir = os.path.abspath(os.curdir)
file = GeneratedFileWithPath(os.path.join(curdir, 'foo',
'bar.manifest'),
'resource bar bar/\ncontent bar bar/')
with errors.context('manifest', 1):
packager.add('foo/bar.manifest', file)
file = GeneratedFileWithPath(os.path.join(curdir, 'foo',
'baz.manifest'),
'resource baz baz/')
with errors.context('manifest', 2):
packager.add('bar/baz.manifest', file)
with errors.context('manifest', 3):
packager.add('qux/qux.manifest',
GeneratedFile(''.join([
'resource qux qux/\n',
'binary-component qux.so\n',
])))
bar_xpt = GeneratedFile('bar.xpt')
qux_xpt = GeneratedFile('qux.xpt')
foo_html = GeneratedFile('foo_html')
bar_html = GeneratedFile('bar_html')
with errors.context('manifest', 4):
packager.add('foo/bar.xpt', bar_xpt)
with errors.context('manifest', 5):
packager.add('foo/bar/foo.html', foo_html)
packager.add('foo/bar/bar.html', bar_html)
file = GeneratedFileWithPath(os.path.join(curdir, 'foo.manifest'),
''.join([
'manifest foo/bar.manifest\n',
'manifest bar/baz.manifest\n',
]))
with errors.context('manifest', 6):
packager.add('foo.manifest', file)
with errors.context('manifest', 7):
packager.add('foo/qux.xpt', qux_xpt)
file = GeneratedFileWithPath(os.path.join(curdir, 'addon',
'chrome.manifest'),
'resource hoge hoge/')
with errors.context('manifest', 8):
packager.add('addon/chrome.manifest', file)
install_rdf = GeneratedFile('<RDF></RDF>')
with errors.context('manifest', 9):
packager.add('addon/install.rdf', install_rdf)
with errors.context('manifest', 10):
packager.add('addon2/install.rdf', install_rdf)
packager.add('addon2/chrome.manifest',
GeneratedFile('binary-component addon2.so'))
with errors.context('manifest', 11):
packager.add('addon3/install.rdf', install_rdf)
packager.add('addon3/chrome.manifest', GeneratedFile(
'manifest components/components.manifest'))
packager.add('addon3/components/components.manifest',
GeneratedFile('binary-component addon3.so'))
with errors.context('manifest', 12):
install_rdf_addon4 = GeneratedFile(
'<RDF>\n<...>\n<em:unpack>true</em:unpack>\n<...>\n</RDF>')
packager.add('addon4/install.rdf', install_rdf_addon4)
with errors.context('manifest', 13):
install_rdf_addon5 = GeneratedFile(
'<RDF>\n<...>\n<em:unpack>false</em:unpack>\n<...>\n</RDF>')
packager.add('addon5/install.rdf', install_rdf_addon5)
with errors.context('manifest', 14):
install_rdf_addon6 = GeneratedFile(
'<RDF>\n<... em:unpack=true>\n<...>\n</RDF>')
packager.add('addon6/install.rdf', install_rdf_addon6)
with errors.context('manifest', 15):
install_rdf_addon7 = GeneratedFile(
'<RDF>\n<... em:unpack=false>\n<...>\n</RDF>')
packager.add('addon7/install.rdf', install_rdf_addon7)
with errors.context('manifest', 16):
install_rdf_addon8 = GeneratedFile(
'<RDF>\n<... em:unpack="true">\n<...>\n</RDF>')
packager.add('addon8/install.rdf', install_rdf_addon8)
with errors.context('manifest', 17):
install_rdf_addon9 = GeneratedFile(
'<RDF>\n<... em:unpack="false">\n<...>\n</RDF>')
packager.add('addon9/install.rdf', install_rdf_addon9)
with errors.context('manifest', 18):
install_rdf_addon10 = GeneratedFile(
'<RDF>\n<... em:unpack=\'true\'>\n<...>\n</RDF>')
packager.add('addon10/install.rdf', install_rdf_addon10)
with errors.context('manifest', 19):
install_rdf_addon11 = GeneratedFile(
'<RDF>\n<... em:unpack=\'false\'>\n<...>\n</RDF>')
packager.add('addon11/install.rdf', install_rdf_addon11)
we_manifest = GeneratedFile(
'{"manifest_version": 2, "name": "Test WebExtension", "version": "1.0"}')
# hybrid and hybrid2 are both bootstrapped extensions with
# embedded webextensions, they differ in the order in which
# the manifests are added to the packager.
with errors.context('manifest', 20):
packager.add('hybrid/install.rdf', install_rdf)
with errors.context('manifest', 21):
packager.add('hybrid/webextension/manifest.json', we_manifest)
with errors.context('manifest', 22):
packager.add('hybrid2/webextension/manifest.json', we_manifest)
with errors.context('manifest', 23):
packager.add('hybrid2/install.rdf', install_rdf)
with errors.context('manifest', 24):
packager.add('webextension/manifest.json', we_manifest)
non_we_manifest = GeneratedFile('{"not a webextension": true}')
with errors.context('manifest', 25):
packager.add('nonwebextension/manifest.json', non_we_manifest)
self.assertEqual(formatter.log, [])
with errors.context('dummy', 1):
packager.close()
self.maxDiff = None
# The formatter is expected to reorder the manifest entries so that
# chrome entries appear before the others.
self.assertEqual(formatter.log, [
(('dummy', 1), 'add_base', '', False),
(('dummy', 1), 'add_base', 'addon', True),
(('dummy', 1), 'add_base', 'addon10', 'unpacked'),
(('dummy', 1), 'add_base', 'addon11', True),
(('dummy', 1), 'add_base', 'addon2', 'unpacked'),
(('dummy', 1), 'add_base', 'addon3', 'unpacked'),
(('dummy', 1), 'add_base', 'addon4', 'unpacked'),
(('dummy', 1), 'add_base', 'addon5', True),
(('dummy', 1), 'add_base', 'addon6', 'unpacked'),
(('dummy', 1), 'add_base', 'addon7', True),
(('dummy', 1), 'add_base', 'addon8', 'unpacked'),
(('dummy', 1), 'add_base', 'addon9', True),
(('dummy', 1), 'add_base', 'hybrid', True),
(('dummy', 1), 'add_base', 'hybrid2', True),
(('dummy', 1), 'add_base', 'qux', False),
(('dummy', 1), 'add_base', 'webextension', True),
((os.path.join(curdir, 'foo', 'bar.manifest'), 2),
'add_manifest', ManifestContent('foo', 'bar', 'bar/')),
((os.path.join(curdir, 'foo', 'bar.manifest'), 1),
'add_manifest', ManifestResource('foo', 'bar', 'bar/')),
(('bar/baz.manifest', 1),
'add_manifest', ManifestResource('bar', 'baz', 'baz/')),
(('qux/qux.manifest', 1),
'add_manifest', ManifestResource('qux', 'qux', 'qux/')),
(('qux/qux.manifest', 2),
'add_manifest', ManifestBinaryComponent('qux', 'qux.so')),
(('manifest', 4), 'add_interfaces', 'foo/bar.xpt', bar_xpt),
(('manifest', 7), 'add_interfaces', 'foo/qux.xpt', qux_xpt),
((os.path.join(curdir, 'addon', 'chrome.manifest'), 1),
'add_manifest', ManifestResource('addon', 'hoge', 'hoge/')),
(('addon2/chrome.manifest', 1), 'add_manifest',
ManifestBinaryComponent('addon2', 'addon2.so')),
(('addon3/components/components.manifest', 1), 'add_manifest',
ManifestBinaryComponent('addon3/components', 'addon3.so')),
(('manifest', 5), 'add', 'foo/bar/foo.html', foo_html),
(('manifest', 5), 'add', 'foo/bar/bar.html', bar_html),
(('manifest', 9), 'add', 'addon/install.rdf', install_rdf),
(('manifest', 10), 'add', 'addon2/install.rdf', install_rdf),
(('manifest', 11), 'add', 'addon3/install.rdf', install_rdf),
(('manifest', 12), 'add', 'addon4/install.rdf',
install_rdf_addon4),
(('manifest', 13), 'add', 'addon5/install.rdf',
install_rdf_addon5),
(('manifest', 14), 'add', 'addon6/install.rdf',
install_rdf_addon6),
(('manifest', 15), 'add', 'addon7/install.rdf',
install_rdf_addon7),
(('manifest', 16), 'add', 'addon8/install.rdf',
install_rdf_addon8),
(('manifest', 17), 'add', 'addon9/install.rdf',
install_rdf_addon9),
(('manifest', 18), 'add', 'addon10/install.rdf',
install_rdf_addon10),
(('manifest', 19), 'add', 'addon11/install.rdf',
install_rdf_addon11),
(('manifest', 20), 'add', 'hybrid/install.rdf', install_rdf),
(('manifest', 21),
'add', 'hybrid/webextension/manifest.json', we_manifest),
(('manifest', 22),
'add', 'hybrid2/webextension/manifest.json', we_manifest),
(('manifest', 23), 'add', 'hybrid2/install.rdf', install_rdf),
(('manifest', 24),
'add', 'webextension/manifest.json', we_manifest),
(('manifest', 25),
'add', 'nonwebextension/manifest.json', non_we_manifest),
])
self.assertEqual(packager.get_bases(),
set(['', 'addon', 'addon2', 'addon3', 'addon4',
'addon5', 'addon6', 'addon7', 'addon8',
'addon9', 'addon10', 'addon11', 'qux',
'hybrid', 'hybrid2', 'webextension']))
self.assertEqual(packager.get_bases(addons=False), set(['', 'qux']))
def create_context_tar(topsrcdir, context_dir, out_path, prefix, args=None):
"""Create a context tarball.
A directory ``context_dir`` containing a Dockerfile will be assembled into
a gzipped tar file at ``out_path``. Files inside the archive will be
prefixed by directory ``prefix``.
We also scan the source Dockerfile for special syntax that influences
context generation.
If a line in the Dockerfile has the form ``# %include <path>``,
the relative path specified on that line will be matched against
files in the source repository and added to the context under the
path ``topsrcdir/``. If an entry is a directory, we add all files
under that directory.
If a line in the Dockerfile has the form ``# %ARG <name>``, occurrences of
the string ``$<name>`` in subsequent lines are replaced with the value
found in the ``args`` argument. Exception: this doesn't apply to VOLUME
definitions.
Returns the SHA-256 hex digest of the created archive.
"""
archive_files = {}
replace = []
for root, dirs, files in os.walk(context_dir):
for f in files:
source_path = os.path.join(root, f)
rel = source_path[len(context_dir) + 1:]
archive_path = os.path.join(prefix, rel)
archive_files[archive_path] = source_path
# Parse Dockerfile for special syntax of extra files to include.
content = []
with open(os.path.join(context_dir, 'Dockerfile'), 'rb') as fh:
for line in fh:
if line.startswith('# %ARG'):
p = line[len('# %ARG '):].strip()
if not args or p not in args:
raise Exception('missing argument: {}'.format(p))
replace.append(
(re.compile(r'\${}\b'.format(p)), args[p].encode('ascii')))
continue
for regexp, s in replace:
line = re.sub(regexp, s, line)
content.append(line)
if not line.startswith('# %include'):
continue
p = line[len('# %include '):].strip()
if os.path.isabs(p):
raise Exception('extra include path cannot be absolute: %s' %
p)
fs_path = os.path.normpath(os.path.join(topsrcdir, p))
# Check for filesystem traversal exploits.
if not fs_path.startswith(topsrcdir):
raise Exception('extra include path outside topsrcdir: %s' % p)
if not os.path.exists(fs_path):
raise Exception('extra include path does not exist: %s' % p)
if os.path.isdir(fs_path):
for root, dirs, files in os.walk(fs_path):
for f in files:
source_path = os.path.join(root, f)
rel = source_path[len(fs_path) + 1:]
archive_path = os.path.join(prefix, 'topsrcdir', p,
rel)
archive_files[archive_path] = source_path
else:
archive_path = os.path.join(prefix, 'topsrcdir', p)
archive_files[archive_path] = fs_path
archive_files[os.path.join(prefix, 'Dockerfile')] = \
GeneratedFile(b''.join(content))
with open(out_path, 'wb') as fh:
create_tar_gz_from_files(fh, archive_files, '%s.tar.gz' % prefix)
h = hashlib.sha256()
with open(out_path, 'rb') as fh:
while True:
data = fh.read(32768)
if not data:
break
h.update(data)
return h.hexdigest()
'bases': {
# base_path: is_addon?
'app': False,
'addon0': True,
},
'manifests': [
ManifestContent('chrome/f', 'oo', 'oo/'),
ManifestContent('chrome/f', 'bar', 'oo/bar/'),
ManifestResource('chrome/f', 'foo', 'resource://bar/'),
ManifestBinaryComponent('components', 'foo.so'),
ManifestContent('app/chrome', 'content', 'foo/'),
ManifestComponent('app/components', '{foo-id}', 'foo.js'),
ManifestContent('addon0/chrome', 'content', 'foo/bar/'),
],
'chrome/f/oo/bar/baz':
GeneratedFile('foobarbaz'),
'chrome/f/oo/baz':
GeneratedFile('foobaz'),
'chrome/f/oo/qux':
GeneratedFile('fooqux'),
'components/foo.so':
GeneratedFile('foo.so'),
'components/foo.xpt':
foo_xpt,
'components/bar.xpt':
bar_xpt,
'foo':
GeneratedFile('foo'),
'app/chrome/foo/foo':
GeneratedFile('appfoo'),
'app/components/foo.js':
def stream_context_tar(topsrcdir, context_dir, out_file, image_name, args):
"""Like create_context_tar, but streams the tar file to the `out_file` file
object."""
archive_files = {}
replace = []
content = []
context_dir = os.path.join(topsrcdir, context_dir)
for root, dirs, files in os.walk(context_dir):
for f in files:
source_path = os.path.join(root, f)
archive_path = source_path[len(context_dir) + 1:]
archive_files[archive_path] = source_path
# Parse Dockerfile for special syntax of extra files to include.
with io.open(os.path.join(context_dir, "Dockerfile"), "r") as fh:
for line in fh:
if line.startswith("# %ARG"):
p = line[len("# %ARG "):].strip()
if not args or p not in args:
raise Exception("missing argument: {}".format(p))
replace.append((re.compile(r"\${}\b".format(p)), args[p]))
continue
for regexp, s in replace:
line = re.sub(regexp, s, line)
content.append(line)
if not line.startswith("# %include"):
continue
p = line[len("# %include "):].strip()
if os.path.isabs(p):
raise Exception("extra include path cannot be absolute: %s" %
p)
fs_path = os.path.normpath(os.path.join(topsrcdir, p))
# Check for filesystem traversal exploits.
if not fs_path.startswith(topsrcdir):
raise Exception("extra include path outside topsrcdir: %s" % p)
if not os.path.exists(fs_path):
raise Exception("extra include path does not exist: %s" % p)
if os.path.isdir(fs_path):
for root, dirs, files in os.walk(fs_path):
for f in files:
source_path = os.path.join(root, f)
rel = source_path[len(fs_path) + 1:]
archive_path = os.path.join("topsrcdir", p, rel)
archive_files[archive_path] = source_path
else:
archive_path = os.path.join("topsrcdir", p)
archive_files[archive_path] = fs_path
archive_files["Dockerfile"] = GeneratedFile(b"".join(
six.ensure_binary(s) for s in content))
writer = HashingWriter(out_file)
create_tar_gz_from_files(writer, archive_files,
"{}.tar".format(image_name))
return writer.hexdigest()
'app': False,
'addon0': 'unpacked',
'addon1': True,
},
'manifests': [
ManifestContent('chrome/f', 'oo', 'oo/'),
ManifestContent('chrome/f', 'bar', 'oo/bar/'),
ManifestResource('chrome/f', 'foo', 'resource://bar/'),
ManifestBinaryComponent('components', 'foo.so'),
ManifestContent('app/chrome', 'content', 'foo/'),
ManifestComponent('app/components', '{foo-id}', 'foo.js'),
ManifestContent('addon0/chrome', 'content', 'foo/bar/'),
ManifestContent('addon1/chrome', 'content', 'foo/bar/'),
],
'files': {
'chrome/f/oo/bar/baz': GeneratedFile('foobarbaz'),
'chrome/f/oo/baz': GeneratedFile('foobaz'),
'chrome/f/oo/qux': GeneratedFile('fooqux'),
'components/foo.so': GeneratedFile('foo.so'),
'components/foo.xpt': foo_xpt,
'components/bar.xpt': bar_xpt,
'foo': GeneratedFile('foo'),
'app/chrome/foo/foo': GeneratedFile('appfoo'),
'app/components/foo.js': GeneratedFile('foo.js'),
'addon0/chrome/foo/bar/baz': GeneratedFile('foobarbaz'),
'addon0/components/foo.xpt': foo2_xpt,
'addon0/components/bar.xpt': bar_xpt,
'addon1/chrome/foo/bar/baz': GeneratedFile('foobarbaz'),
'addon1/components/foo.xpt': foo2_xpt,
'addon1/components/bar.xpt': bar_xpt,
},
def test_l10n_repack(self):
foo = GeneratedFile('foo')
foobar = GeneratedFile('foobar')
qux = GeneratedFile('qux')
bar = GeneratedFile('bar')
baz = GeneratedFile('baz')
dict_aa = GeneratedFile('dict_aa')
dict_bb = GeneratedFile('dict_bb')
dict_cc = GeneratedFile('dict_cc')
barbaz = GeneratedFile('barbaz')
lst = GeneratedFile('foo\nbar')
app_finder = MockFinder({
'bar/foo':
foo,
'chrome/foo/foobar':
foobar,
'chrome/qux/qux.properties':
qux,
'chrome/qux/baz/baz.properties':
baz,
'chrome/chrome.manifest':
ManifestFile('chrome', [
ManifestContent('chrome', 'foo', 'foo/'),
ManifestLocale('chrome', 'qux', 'en-US', 'qux/'),
]),
'chrome.manifest':
ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]),
'dict/aa':
dict_aa,
'app/chrome/bar/barbaz.dtd':
barbaz,
'app/chrome/chrome.manifest':
ManifestFile(
'app/chrome',
[ManifestLocale('app/chrome', 'bar', 'en-US', 'bar/')]),
'app/chrome.manifest':
ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]),
'app/dict/bb':
dict_bb,
'app/dict/cc':
dict_cc,
'app/chrome/bar/search/foo.xml':
foo,
'app/chrome/bar/search/bar.xml':
bar,
'app/chrome/bar/search/lst.txt':
lst,
})
app_finder.jarlogs = {}
app_finder.base = 'app'
foo_l10n = GeneratedFile('foo_l10n')
qux_l10n = GeneratedFile('qux_l10n')
baz_l10n = GeneratedFile('baz_l10n')
barbaz_l10n = GeneratedFile('barbaz_l10n')
lst_l10n = GeneratedFile('foo\nqux')
l10n_finder = MockFinder({
'chrome/qux-l10n/qux.properties':
qux_l10n,
'chrome/qux-l10n/baz/baz.properties':
baz_l10n,
'chrome/chrome.manifest':
ManifestFile('chrome', [
ManifestLocale('chrome', 'qux', 'x-test', 'qux-l10n/'),
]),
'chrome.manifest':
ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]),
'dict/bb':
dict_bb,
'dict/cc':
dict_cc,
'app/chrome/bar-l10n/barbaz.dtd':
barbaz_l10n,
'app/chrome/chrome.manifest':
ManifestFile(
'app/chrome',
[ManifestLocale('app/chrome', 'bar', 'x-test', 'bar-l10n/')]),
'app/chrome.manifest':
ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]),
'app/dict/aa':
dict_aa,
'app/chrome/bar-l10n/search/foo.xml':
foo_l10n,
'app/chrome/bar-l10n/search/qux.xml':
qux_l10n,
'app/chrome/bar-l10n/search/lst.txt':
lst_l10n,
})
l10n_finder.base = 'l10n'
copier = FileRegistry()
formatter = FlatFormatter(copier)
l10n._repack(app_finder, l10n_finder, copier, formatter,
['dict', 'chrome/**/search/*.xml'])
self.maxDiff = None
repacked = {
'bar/foo':
foo,
'chrome/foo/foobar':
foobar,
'chrome/qux-l10n/qux.properties':
qux_l10n,
'chrome/qux-l10n/baz/baz.properties':
baz_l10n,
'chrome/chrome.manifest':
ManifestFile('chrome', [
ManifestContent('chrome', 'foo', 'foo/'),
ManifestLocale('chrome', 'qux', 'x-test', 'qux-l10n/'),
]),
'chrome.manifest':
ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]),
'dict/bb':
dict_bb,
'dict/cc':
dict_cc,
'app/chrome/bar-l10n/barbaz.dtd':
barbaz_l10n,
'app/chrome/chrome.manifest':
ManifestFile(
'app/chrome',
[ManifestLocale('app/chrome', 'bar', 'x-test', 'bar-l10n/')]),
'app/chrome.manifest':
ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]),
'app/dict/aa':
dict_aa,
'app/chrome/bar-l10n/search/foo.xml':
foo_l10n,
'app/chrome/bar-l10n/search/qux.xml':
qux_l10n,
'app/chrome/bar-l10n/search/lst.txt':
lst_l10n,
}
self.assertEqual(
dict((p, f.open().read()) for p, f in copier),
dict((p, f.open().read()) for p, f in repacked.iteritems()))
def test_simple_packager(self):
class GeneratedFileWithPath(GeneratedFile):
def __init__(self, path, content):
GeneratedFile.__init__(self, content)
self.path = path
formatter = MockFormatter()
packager = SimplePackager(formatter)
curdir = os.path.abspath(os.curdir)
file = GeneratedFileWithPath(
os.path.join(curdir, 'foo', 'bar.manifest'),
'resource bar bar/\ncontent bar bar/')
with errors.context('manifest', 1):
packager.add('foo/bar.manifest', file)
file = GeneratedFileWithPath(
os.path.join(curdir, 'foo', 'baz.manifest'), 'resource baz baz/')
with errors.context('manifest', 2):
packager.add('bar/baz.manifest', file)
with errors.context('manifest', 3):
packager.add('qux/qux.manifest',
GeneratedFile('resource qux qux/'))
bar_xpt = GeneratedFile('bar.xpt')
qux_xpt = GeneratedFile('qux.xpt')
foo_html = GeneratedFile('foo_html')
bar_html = GeneratedFile('bar_html')
with errors.context('manifest', 4):
packager.add('foo/bar.xpt', bar_xpt)
with errors.context('manifest', 5):
packager.add('foo/bar/foo.html', foo_html)
packager.add('foo/bar/bar.html', bar_html)
file = GeneratedFileWithPath(
os.path.join(curdir, 'foo.manifest'), ''.join([
'manifest foo/bar.manifest\n',
'manifest bar/baz.manifest\n',
]))
with errors.context('manifest', 6):
packager.add('foo.manifest', file)
with errors.context('manifest', 7):
packager.add('foo/qux.xpt', qux_xpt)
self.assertEqual(formatter.log, [])
with errors.context('dummy', 1):
packager.close()
self.maxDiff = None
self.assertEqual(formatter.log, [
(('dummy', 1), 'add_base', 'qux'),
((os.path.join(curdir, 'foo', 'bar.manifest'), 1), 'add_manifest',
ManifestResource('foo', 'bar', 'bar/')),
((os.path.join(curdir, 'foo', 'bar.manifest'), 2), 'add_manifest',
ManifestContent('foo', 'bar', 'bar/')),
(('bar/baz.manifest', 1), 'add_manifest',
ManifestResource('bar', 'baz', 'baz/')),
(('qux/qux.manifest', 1), 'add_manifest',
ManifestResource('qux', 'qux', 'qux/')),
(('manifest', 4), 'add_interfaces', 'foo/bar.xpt', bar_xpt),
(('manifest', 7), 'add_interfaces', 'foo/qux.xpt', qux_xpt),
(('manifest', 5), 'add', 'foo/bar/foo.html', foo_html),
(('manifest', 5), 'add', 'foo/bar/bar.html', bar_html),
])
self.assertEqual(packager.get_bases(), set(['', 'qux']))
def test_simple_packager_manifest_consistency(self):
formatter = MockFormatter()
# bar/ is detected as an addon because of install.rdf, but top-level
# includes a manifest inside bar/.
packager = SimplePackager(formatter)
packager.add('base.manifest', GeneratedFile(
'manifest foo/bar.manifest\n'
'manifest bar/baz.manifest\n'
))
packager.add('foo/bar.manifest', GeneratedFile('resource bar bar'))
packager.add('bar/baz.manifest', GeneratedFile('resource baz baz'))
packager.add('bar/install.rdf', GeneratedFile(''))
with self.assertRaises(ErrorMessage) as e:
packager.close()
self.assertEqual(e.exception.message,
'Error: "bar/baz.manifest" is included from "base.manifest", '
'which is outside "bar"')
# bar/ is detected as a separate base because of chrome.manifest that
# is included nowhere, but top-level includes another manifest inside
# bar/.
packager = SimplePackager(formatter)
packager.add('base.manifest', GeneratedFile(
'manifest foo/bar.manifest\n'
'manifest bar/baz.manifest\n'
))
packager.add('foo/bar.manifest', GeneratedFile('resource bar bar'))
packager.add('bar/baz.manifest', GeneratedFile('resource baz baz'))
packager.add('bar/chrome.manifest', GeneratedFile('resource baz baz'))
with self.assertRaises(ErrorMessage) as e:
packager.close()
self.assertEqual(e.exception.message,
'Error: "bar/baz.manifest" is included from "base.manifest", '
'which is outside "bar"')
# bar/ is detected as a separate base because of chrome.manifest that
# is included nowhere, but chrome.manifest includes baz.manifest from
# the same directory. This shouldn't error out.
packager = SimplePackager(formatter)
packager.add('base.manifest', GeneratedFile(
'manifest foo/bar.manifest\n'
))
packager.add('foo/bar.manifest', GeneratedFile('resource bar bar'))
packager.add('bar/baz.manifest', GeneratedFile('resource baz baz'))
packager.add('bar/chrome.manifest',
GeneratedFile('manifest baz.manifest'))
packager.close()
"app/chrome/addons/addon2": True,
},
"manifests": [
ManifestContent("chrome/f", "oo", "oo/"),
ManifestContent("chrome/f", "bar", "oo/bar/"),
ManifestResource("chrome/f", "foo", "resource://bar/"),
ManifestBinaryComponent("components", "foo.so"),
ManifestContent("app/chrome", "content", "foo/"),
ManifestComponent("app/components", "{foo-id}", "foo.js"),
ManifestContent("addon0/chrome", "addon0", "foo/bar/"),
ManifestContent("addon1/chrome", "addon1", "foo/bar/"),
ManifestContent("app/chrome/addons/addon2/chrome", "addon2",
"foo/bar/"),
],
"files": {
"chrome/f/oo/bar/baz": GeneratedFile(b"foobarbaz"),
"chrome/f/oo/baz": GeneratedFile(b"foobaz"),
"chrome/f/oo/qux": GeneratedFile(b"fooqux"),
"components/foo.so": GeneratedFile(b"foo.so"),
"components/foo.xpt": foo_xpt,
"components/bar.xpt": bar_xpt,
"foo": GeneratedFile(b"foo"),
"app/chrome/foo/foo": GeneratedFile(b"appfoo"),
"app/components/foo.js": GeneratedFile(b"foo.js"),
"addon0/chrome/foo/bar/baz": GeneratedFile(b"foobarbaz"),
"addon0/components/foo.xpt": foo2_xpt,
"addon0/components/bar.xpt": bar_xpt,
"addon1/chrome/foo/bar/baz": GeneratedFile(b"foobarbaz"),
"addon1/components/foo.xpt": foo2_xpt,
"addon1/components/bar.xpt": bar_xpt,
"app/chrome/addons/addon2/chrome/foo/bar/baz":
def stream_context_tar(topsrcdir, context_dir, out_file, prefix, args=None):
"""Like create_context_tar, but streams the tar file to the `out_file` file
object."""
archive_files = {}
replace = []
content = []
context_dir = os.path.join(topsrcdir, context_dir)
for root, dirs, files in os.walk(context_dir):
for f in files:
source_path = os.path.join(root, f)
rel = source_path[len(context_dir) + 1:]
archive_path = os.path.join(prefix, rel)
archive_files[archive_path] = source_path
# Parse Dockerfile for special syntax of extra files to include.
with open(os.path.join(context_dir, 'Dockerfile'), 'rb') as fh:
for line in fh:
if line.startswith('# %ARG'):
p = line[len('# %ARG '):].strip()
if not args or p not in args:
raise Exception('missing argument: {}'.format(p))
replace.append((re.compile(r'\${}\b'.format(p)),
args[p].encode('ascii')))
continue
for regexp, s in replace:
line = re.sub(regexp, s, line)
content.append(line)
if not line.startswith('# %include'):
continue
p = line[len('# %include '):].strip()
if os.path.isabs(p):
raise Exception('extra include path cannot be absolute: %s' % p)
fs_path = os.path.normpath(os.path.join(topsrcdir, p))
# Check for filesystem traversal exploits.
if not fs_path.startswith(topsrcdir):
raise Exception('extra include path outside topsrcdir: %s' % p)
if not os.path.exists(fs_path):
raise Exception('extra include path does not exist: %s' % p)
if os.path.isdir(fs_path):
for root, dirs, files in os.walk(fs_path):
for f in files:
source_path = os.path.join(root, f)
rel = source_path[len(fs_path) + 1:]
archive_path = os.path.join(prefix, 'topsrcdir', p, rel)
archive_files[archive_path] = source_path
else:
archive_path = os.path.join(prefix, 'topsrcdir', p)
archive_files[archive_path] = fs_path
archive_files[os.path.join(prefix, 'Dockerfile')] = \
GeneratedFile(b''.join(content))
writer = HashingWriter(out_file)
create_tar_gz_from_files(writer, archive_files, '%s.tar.gz' % prefix)
return writer.hexdigest()
def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()):
app = LocaleManifestFinder(app_finder)
l10n = LocaleManifestFinder(l10n_finder)
# The code further below assumes there's only one locale replaced with
# another one.
if len(app.locales) > 1:
errors.fatal("Multiple app locales aren't supported: " +
",".join(app.locales))
if len(l10n.locales) > 1:
errors.fatal("Multiple l10n locales aren't supported: " +
",".join(l10n.locales))
locale = app.locales[0]
l10n_locale = l10n.locales[0]
# For each base directory, store what path a locale chrome package name
# corresponds to.
# e.g., for the following entry under app/chrome:
# locale foo en-US path/to/files
# keep track that the locale path for foo in app is
# app/chrome/path/to/files.
# As there may be multiple locale entries with the same base, but with
# different flags, that tracking takes the flags into account when there
# are some. Example:
# locale foo en-US path/to/files/win os=Win
# locale foo en-US path/to/files/mac os=Darwin
def key(entry):
if entry.flags:
return '%s %s' % (entry.name, entry.flags)
return entry.name
l10n_paths = {}
for e in l10n.entries:
if isinstance(e, ManifestChrome):
base = mozpath.basedir(e.path, app.bases)
l10n_paths.setdefault(base, {})
l10n_paths[base][key(e)] = e.path
# For chrome and non chrome files or directories, store what langpack path
# corresponds to a package path.
paths = {}
for e in app.entries:
if isinstance(e, ManifestEntryWithRelPath):
base = mozpath.basedir(e.path, app.bases)
if base not in l10n_paths:
errors.fatal("Locale doesn't contain %s/" % base)
# Allow errors to accumulate
continue
if key(e) not in l10n_paths[base]:
errors.fatal("Locale doesn't have a manifest entry for '%s'" %
e.name)
# Allow errors to accumulate
continue
paths[e.path] = l10n_paths[base][key(e)]
for pattern in non_chrome:
for base in app.bases:
path = mozpath.join(base, pattern)
left = set(p for p, f in app_finder.find(path))
right = set(p for p, f in l10n_finder.find(path))
for p in right:
paths[p] = p
for p in left - right:
paths[p] = None
# Create a new package, with non localized bits coming from the original
# package, and localized bits coming from the langpack.
packager = SimplePackager(formatter)
built_in_addons = None
for p, f in app_finder:
if is_manifest(p):
# Remove localized manifest entries.
for e in [e for e in f if e.localized]:
f.remove(e)
# If the path is one that needs a locale replacement, use the
# corresponding file from the langpack.
path = None
if p in paths:
path = paths[p]
if not path:
continue
else:
base = mozpath.basedir(p, paths.keys())
if base:
subpath = mozpath.relpath(p, base)
path = mozpath.normpath(mozpath.join(paths[base], subpath))
if path:
files = [f for p, f in l10n_finder.find(path)]
if not len(files):
if base not in non_chrome:
finderBase = ""
if hasattr(l10n_finder, 'base'):
finderBase = l10n_finder.base
errors.error("Missing file: %s" %
os.path.join(finderBase, path))
else:
packager.add(path, files[0])
elif p.endswith('built_in_addons.json'):
built_in_addons = (p, f)
else:
packager.add(p, f)
# Add localized manifest entries from the langpack.
l10n_manifests = []
for base in set(e.base for e in l10n.entries):
m = ManifestFile(base, [e for e in l10n.entries if e.base == base])
path = mozpath.join(base, 'chrome.%s.manifest' % l10n_locale)
l10n_manifests.append((path, m))
bases = packager.get_bases()
for path, m in l10n_manifests:
base = mozpath.basedir(path, bases)
packager.add(path, m)
# Add a "manifest $path" entry in the top manifest under that base.
m = ManifestFile(base)
m.add(Manifest(base, mozpath.relpath(path, base)))
packager.add(mozpath.join(base, 'chrome.manifest'), m)
packager.close()
dictionaries = {}
# Add any remaining non chrome files.
for pattern in non_chrome:
for base in bases:
for p, f in l10n_finder.find(mozpath.join(base, pattern)):
if not formatter.contains(p):
if p.startswith('dictionaries/') and p.endswith('.dic'):
base, ext = os.path.splitext(os.path.basename(p))
dictionaries[base] = p
formatter.add(p, f)
# Update the built-in add-ons manifest with the new list of dictionaries
# from the langpack.
if built_in_addons:
data = json.load(built_in_addons[1].open())
data['dictionaries'] = dictionaries
formatter.add(built_in_addons[0], GeneratedFile(json.dumps(data)))
# Resources in `localization` directories are packaged from the source and then
# if localized versions are present in the l10n dir, we package them as well
# keeping the source dir resources as a runtime fallback.
for p, f in l10n_finder.find('**/localization'):
if not formatter.contains(p):
formatter.add(p, f)
# Transplant jar preloading information.
for path, log in app_finder.jarlogs.iteritems():
assert isinstance(copier[path], Jarrer)
copier[path].preload([l.replace(locale, l10n_locale) for l in log])
# Compiled typelib for the following IDL:
# interface foo;
# [scriptable, uuid(5f70da76-519c-4858-b71e-e3c92333e2d6)]
# interface bar {
# void bar(in foo f);
# };
# We need to make this [scriptable] so it doesn't get deleted from the
# typelib. We don't need to make the foo interfaces below [scriptable],
# because they will be automatically included by virtue of being an
# argument to a method of |bar|.
bar_xpt = GeneratedFile(
b"\x58\x50\x43\x4F\x4D\x0A\x54\x79\x70\x65\x4C\x69\x62\x0D\x0A\x1A" +
b"\x01\x02\x00\x02\x00\x00\x00\x7B\x00\x00\x00\x24\x00\x00\x00\x5C" +
b"\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" +
b"\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x5F" +
b"\x70\xDA\x76\x51\x9C\x48\x58\xB7\x1E\xE3\xC9\x23\x33\xE2\xD6\x00" +
b"\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x0D\x00\x66\x6F\x6F\x00" +
b"\x62\x61\x72\x00\x62\x61\x72\x00\x00\x00\x00\x01\x00\x00\x00\x00" +
b"\x09\x01\x80\x92\x00\x01\x80\x06\x00\x00\x80")
# Compiled typelib for the following IDL:
# [uuid(3271bebc-927e-4bef-935e-44e0aaf3c1e5)]
# interface foo {
# void foo();
# };
foo_xpt = GeneratedFile(
b"\x58\x50\x43\x4F\x4D\x0A\x54\x79\x70\x65\x4C\x69\x62\x0D\x0A\x1A" +
b"\x01\x02\x00\x01\x00\x00\x00\x57\x00\x00\x00\x24\x00\x00\x00\x40" +
b"\x80\x00\x00\x32\x71\xBE\xBC\x92\x7E\x4B\xEF\x93\x5E\x44\xE0\xAA" +
b"\xF3\xC1\xE5\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x09\x00" +
def create_registry(self):
registry = FileRegistry()
registry.add('foo/bar', GeneratedFile('foo/bar'))
registry.add('baz/qux', GeneratedFile('baz/qux'))
return FileRegistrySubtree('base/root', registry)
