def test_populate_registry(self):
m = self._get_test_manifest()
r = FileRegistry()
m.populate_registry(r)
self.assertEqual(len(r), 3)
self.assertEqual(r.paths(), ["c_dest", "e_dest", "s_dest"])
def test_populate_registry(self):
m = self._get_test_manifest()
r = FileRegistry()
m.populate_registry(r)
self.assertEqual(len(r), 4)
self.assertEqual(r.paths(), ['c_dest', 'e_dest', 'o_dest', 's_dest'])
class IDLUpdater(object):
'''
Updates interfaces uuids in IDL files.
'''
def __init__(self, interfaces):
from mozpack.copier import FileRegistry
self.interfaces = interfaces;
self.registry = FileRegistry()
def add(self, name):
for interface in self.interfaces.get_by_name(name):
self._add(interface)
def _add(self, interface):
from mozpack.files import GeneratedFile
from uuid import uuid4
path = interface.filename
if not self.registry.contains(path):
self.registry.add(path, GeneratedFile(open(path).read()))
content = self.registry[path].content
content = content[:interface.uuid_pos] + str(uuid4()) + \
content[interface.uuid_pos + len(interface.uuid):]
self.registry[path].content = content
# Recurse through all the interfaces deriving from this one
for derived in self.interfaces.get_by_base(interface.name):
self._add(derived)
def update(self):
for p, f in self.registry:
f.copy(p)
def test_jar_formatter(self):
registry = FileRegistry()
formatter = JarFormatter(registry)
formatter.add_manifest(ManifestContent('f', 'oo', 'oo/'))
formatter.add_manifest(ManifestContent('f', 'bar', 'oo/bar/'))
formatter.add('f/oo/bar/baz', GeneratedFile('foobarbaz'))
formatter.add('f/oo/qux', GeneratedFile('fooqux'))
self.assertEqual(registry.paths(),
['chrome.manifest', 'f/f.manifest', 'f/oo.jar'])
self.assertEqual(registry['chrome.manifest'].open().read(),
'manifest f/f.manifest\n')
self.assertEqual(registry['f/f.manifest'].open().read(), ''.join([
'content oo jar:oo.jar!/\n',
'content bar jar:oo.jar!/bar/\n',
]))
self.assertTrue(formatter.contains('f/oo/bar/baz'))
self.assertFalse(formatter.contains('foo/bar/baz'))
self.assertEqual(registry['f/oo.jar'].paths(), ['bar/baz', 'qux'])
formatter.add_manifest(ManifestResource('f', 'foo', 'resource://bar/'))
self.assertEqual(registry['f/f.manifest'].open().read(), ''.join([
'content oo jar:oo.jar!/\n',
'content bar jar:oo.jar!/bar/\n',
'resource foo resource://bar/\n',
]))
def test_partial_paths(self):
cases = {
'foo/bar/baz/zot': ['foo/bar/baz', 'foo/bar', 'foo'],
'foo/bar': ['foo'],
'bar': [],
}
reg = FileRegistry()
for path, parts in cases.iteritems():
self.assertEqual(reg._partial_paths(path), parts)
def test_flat_formatter(self):
registry = FileRegistry()
formatter = FlatFormatter(registry)
formatter.add_base('app')
formatter.add('f/oo/bar', GeneratedFile('foobar'))
formatter.add('f/oo/baz', GeneratedFile('foobaz'))
formatter.add('f/oo/qux', GeneratedFile('fooqux'))
formatter.add_manifest(ManifestContent('f/oo', 'bar', 'bar'))
formatter.add_manifest(ManifestContent('f/oo', 'qux', 'qux'))
self.assertEqual(registry.paths(),
['f/oo/bar', 'f/oo/baz', 'f/oo/qux',
'chrome.manifest', 'f/f.manifest',
'f/oo/oo.manifest'])
self.assertEqual(registry['chrome.manifest'].open().read(),
'manifest f/f.manifest\n')
self.assertEqual(registry['f/f.manifest'].open().read(),
'manifest oo/oo.manifest\n')
self.assertEqual(registry['f/oo/oo.manifest'].open().read(), ''.join([
'content bar bar\n',
'content qux qux\n',
]))
formatter.add_interfaces('components/foo.xpt', foo_xpt)
formatter.add_interfaces('components/bar.xpt', bar_xpt)
self.assertEqual(registry.paths(),
['f/oo/bar', 'f/oo/baz', 'f/oo/qux',
'chrome.manifest', 'f/f.manifest',
'f/oo/oo.manifest', 'components/components.manifest',
'components/interfaces.xpt'])
self.assertEqual(registry['chrome.manifest'].open().read(), ''.join([
'manifest f/f.manifest\n',
'manifest components/components.manifest\n',
]))
self.assertEqual(
registry['components/components.manifest'].open().read(),
'interfaces interfaces.xpt\n'
)
registry['components/interfaces.xpt'] \
.copy(self.tmppath('interfaces.xpt'))
linked = read_interfaces(self.tmppath('interfaces.xpt'))
foo = read_interfaces(foo_xpt.open())
bar = read_interfaces(bar_xpt.open())
self.assertEqual(foo['foo'], linked['foo'])
self.assertEqual(bar['bar'], linked['bar'])
formatter.add_manifest(ManifestContent('app/chrome', 'content',
'foo/'))
self.assertEqual(registry['chrome.manifest'].open().read(), ''.join([
'manifest f/f.manifest\n',
'manifest components/components.manifest\n',
]))
self.assertEqual(registry['app/chrome.manifest'].open().read(),
'manifest chrome/chrome.manifest\n')
self.assertEqual(registry['app/chrome/chrome.manifest'].open().read(),
'content content foo/\n')
def find_generated_harness_files():
# TEST_HARNESS_FILES end up in an install manifest at
# $topsrcdir/_build_manifests/install/_tests.
manifest = InstallManifest(mozpath.join(buildconfig.topobjdir,
'_build_manifests',
'install',
'_tests'))
registry = FileRegistry()
manifest.populate_registry(registry)
# Conveniently, the generated files we care about will already
# exist in the objdir, so we can identify relevant files if
# they're an `ExistingFile` instance.
return [mozpath.join('_tests', p) for p in registry.paths()
if isinstance(registry[p], ExistingFile)]
def test_required_directories(self):
self.registry = FileRegistry()
self.registry.add('foo', GeneratedFile('foo'))
self.assertEqual(self.registry.required_directories(), set())
self.registry.add('bar/baz', GeneratedFile('barbaz'))
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.add('bar/zot', GeneratedFile('barzot'))
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.add('bar/zap/zot', GeneratedFile('barzapzot'))
self.assertEqual(self.registry.required_directories(), {'bar', 'bar/zap'})
self.registry.remove('bar/zap/zot')
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.remove('bar/baz')
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.remove('bar/zot')
self.assertEqual(self.registry.required_directories(), set())
self.registry.add('x/y/z', GeneratedFile('xyz'))
self.assertEqual(self.registry.required_directories(), {'x', 'x/y'})
def test_registry_paths(self):
self.registry = FileRegistry()
# Can't add a file if it requires a directory in place of a
# file we also require.
self.registry.add('foo', GeneratedFile('foo'))
self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar',
GeneratedFile('foobar'))
# Can't add a file if we already have a directory there.
self.registry.add('bar/baz', GeneratedFile('barbaz'))
self.assertRaises(ErrorMessage, self.registry.add, 'bar',
GeneratedFile('bar'))
# Bump the count of things that require bar/ to 2.
self.registry.add('bar/zot', GeneratedFile('barzot'))
self.assertRaises(ErrorMessage, self.registry.add, 'bar',
GeneratedFile('bar'))
# Drop the count of things that require bar/ to 1.
self.registry.remove('bar/baz')
self.assertRaises(ErrorMessage, self.registry.add, 'bar',
GeneratedFile('bar'))
# Drop the count of things that require bar/ to 0.
self.registry.remove('bar/zot')
self.registry.add('bar/zot', GeneratedFile('barzot'))
def test_required_directories(self):
self.registry = FileRegistry()
self.registry.add("foo", GeneratedFile("foo"))
self.assertEqual(self.registry.required_directories(), set())
self.registry.add("bar/baz", GeneratedFile("barbaz"))
self.assertEqual(self.registry.required_directories(), {"bar"})
self.registry.add("bar/zot", GeneratedFile("barzot"))
self.assertEqual(self.registry.required_directories(), {"bar"})
self.registry.add("bar/zap/zot", GeneratedFile("barzapzot"))
self.assertEqual(self.registry.required_directories(), {"bar", "bar/zap"})
self.registry.remove("bar/zap/zot")
self.assertEqual(self.registry.required_directories(), {"bar"})
self.registry.remove("bar/baz")
self.assertEqual(self.registry.required_directories(), {"bar"})
self.registry.remove("bar/zot")
self.assertEqual(self.registry.required_directories(), set())
self.registry.add("x/y/z", GeneratedFile("xyz"))
self.assertEqual(self.registry.required_directories(), {"x", "x/y"})
def process_manifest(
destdir,
paths,
track=None,
remove_unaccounted=True,
remove_all_directory_symlinks=True,
remove_empty_directories=True,
defines={},
):
if track:
if os.path.exists(track):
# We use the same format as install manifests for the tracking
# data.
manifest = InstallManifest(path=track)
remove_unaccounted = FileRegistry()
dummy_file = BaseFile()
finder = FileFinder(destdir, find_executables=False, find_dotfiles=True)
for dest in manifest._dests:
for p, f in finder.find(dest):
remove_unaccounted.add(p, dummy_file)
else:
# If tracking is enabled and there is no file, we don't want to
# be removing anything.
remove_unaccounted = False
remove_empty_directories = False
remove_all_directory_symlinks = False
manifest = InstallManifest()
for path in paths:
manifest |= InstallManifest(path=path)
copier = FileCopier()
manifest.populate_registry(copier, defines_override=defines)
result = copier.copy(
destdir,
remove_unaccounted=remove_unaccounted,
remove_all_directory_symlinks=remove_all_directory_symlinks,
remove_empty_directories=remove_empty_directories,
)
if track:
manifest.write(path=track)
return result
def __init__(self, finders):
# Can't import globally, because of the dependency of mozpack.copier
# on this module.
from mozpack.copier import FileRegistry
self.files = FileRegistry()
for base, finder in sorted(finders.iteritems()):
if self.files.contains(base):
self.files.remove(base)
for p, f in finder.find(''):
self.files.add(mozpath.join(base, p), f)
def _unpack_test(self, cls):
# Format a package with the given formatter class
copier = self._get_copier(cls)
copier.copy(self.tmpdir)
# Unpack that package. Its content is expected to match that of a Flat
# formatted package.
registry = FileRegistry()
unpack_to_registry(self.tmpdir, registry,
getattr(cls, 'OMNIJAR_NAME', None))
self.assertEqual(get_contents(registry, read_all=True, mode='rb'),
self.contents)
def make_file_mapping(install_manifests):
file_mapping = {}
for manifest, destination in install_manifests:
destination = os.path.abspath(destination)
reg = FileRegistry()
manifest.populate_registry(reg)
for dst, src in reg:
if hasattr(src, 'path'):
# Any paths that get compared to source file names need to go through normpath.
abs_dest = normpath(os.path.join(destination, dst))
file_mapping[abs_dest] = normpath(src.path)
return file_mapping
def test_final_target_files_wildcard(self):
"""Ensure that wildcards in FINAL_TARGET_FILES work properly."""
env = self._consume('final-target-files-wildcard', FasterMakeBackend)
m = InstallManifest(
path=mozpath.join(env.topobjdir, 'faster', 'install_dist_bin'))
self.assertEqual(len(m), 1)
reg = FileRegistry()
m.populate_registry(reg)
expected = [('foo/bar.xyz', 'bar.xyz'), ('foo/foo.xyz', 'foo.xyz')]
actual = [(path, mozpath.relpath(f.path, env.topsrcdir))
for (path, f) in reg]
self.assertEqual(expected, actual)
def _load_manifest(self, path, root):
install_manifest = InstallManifest(path)
reg = FileRegistry()
install_manifest.populate_registry(reg)
for dest, src in reg:
if hasattr(src, 'path'):
if not os.path.isabs(dest):
dest = root + dest
self._install_mapping[dest] = (src.path,
isinstance(
src, PreprocessedFile))
class ComposedFinder(BaseFinder):
'''
Composes multiple File Finders in some sort of virtual file system.
A ComposedFinder is initialized from a dictionary associating paths to
*Finder instances.
Note this could be optimized to be smarter than getting all the files
in advance.
'''
def __init__(self, finders):
# Can't import globally, because of the dependency of mozpack.copier
# on this module.
from mozpack.copier import FileRegistry
self.files = FileRegistry()
for base, finder in sorted(finders.iteritems()):
if self.files.contains(base):
self.files.remove(base)
for p, f in finder.find(''):
self.files.add(mozpath.join(base, p), f)
def find(self, pattern):
for p in self.files.match(pattern):
yield p, self.files[p]
class ComposedFinder(BaseFinder):
'''
Composes multiple File Finders in some sort of virtual file system.
A ComposedFinder is initialized from a dictionary associating paths to
*Finder instances.
Note this could be optimized to be smarter than getting all the files
in advance.
'''
def __init__(self, finders):
# Can't import globally, because of the dependency of mozpack.copier
# on this module.
from mozpack.copier import FileRegistry
self.files = FileRegistry()
for base, finder in sorted(finders.iteritems()):
if self.files.contains(base):
self.files.remove(base)
for p, f in finder.find(''):
self.files.add(mozpath.join(base, p), f)
def find(self, pattern):
for p in self.files.match(pattern):
yield p, self.files[p]
def test_file_registry(self):
self.registry = FileRegistry()
self.registry.add('foo', GeneratedFile('foo'))
bar = GeneratedFile('bar')
self.registry.add('bar', bar)
self.assertEqual(self.registry.paths(), ['foo', 'bar'])
self.assertEqual(self.registry['bar'], bar)
self.assertRaises(ErrorMessage, self.registry.add, 'foo',
GeneratedFile('foo2'))
self.assertRaises(ErrorMessage, self.registry.remove, 'qux')
self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar',
GeneratedFile('foobar'))
self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar/baz',
GeneratedFile('foobar'))
self.assertEqual(self.registry.paths(), ['foo', 'bar'])
self.registry.remove('foo')
self.assertEqual(self.registry.paths(), ['bar'])
self.registry.remove('bar')
self.assertEqual(self.registry.paths(), [])
self.prepare_match_test()
self.do_match_test()
self.assertTrue(self.checked)
self.assertEqual(self.registry.paths(), [
'bar',
'foo/bar',
'foo/baz',
'foo/qux/1',
'foo/qux/bar',
'foo/qux/2/test',
'foo/qux/2/test2',
])
self.registry.remove('foo/qux')
self.assertEqual(self.registry.paths(), ['bar', 'foo/bar', 'foo/baz'])
self.registry.add('foo/qux', GeneratedFile('fooqux'))
self.assertEqual(self.registry.paths(), ['bar', 'foo/bar', 'foo/baz',
'foo/qux'])
self.registry.remove('foo/b*')
self.assertEqual(self.registry.paths(), ['bar', 'foo/qux'])
self.assertEqual([f for f, c in self.registry], ['bar', 'foo/qux'])
self.assertEqual(len(self.registry), 2)
self.add('foo/.foo')
self.assertTrue(self.registry.contains('foo/.foo'))
def get_registry(paths):
used_paths = set()
registry = FileRegistry()
for base, path in paths:
full_path = mozpath.join(buildconfig.topobjdir, path)
if not os.path.exists(full_path):
continue
used_paths.add(full_path)
reg = FileRegistry()
InstallManifest(full_path).populate_registry(reg)
for p, f in reg:
path = mozpath.join(base, p)
try:
registry.add(path, f)
except Exception:
pass
return registry, used_paths
def test_bases(self):
formatter = FlatFormatter(FileRegistry())
formatter.add_base("")
formatter.add_base("addon0", addon=True)
formatter.add_base("browser")
self.assertEqual(formatter._get_base("platform.ini"),
("", "platform.ini"))
self.assertEqual(
formatter._get_base("browser/application.ini"),
("browser", "application.ini"),
)
self.assertEqual(formatter._get_base("addon0/install.rdf"),
("addon0", "install.rdf"))
def __init__(self, source):
if isinstance(source, BaseFinder):
self._finder = source
else:
self._finder = FileFinder(source)
self.base = self._finder.base
self.files = FileRegistry()
self.kind = 'flat'
self.omnijar = None
self.jarlogs = {}
self.optimizedjars = False
self.compressed = True
jars = set()
for p, f in self._finder.find('*'):
# Skip the precomplete file, which is generated at packaging time.
if p == 'precomplete':
continue
base = mozpath.dirname(p)
# If the file is a zip/jar that is not a .xpi, and contains a
# chrome.manifest, it is an omnijar. All the files it contains
# go in the directory containing the omnijar. Manifests are merged
# if there is a corresponding manifest in the directory.
if not p.endswith('.xpi') and self._maybe_zip(f) and \
(mozpath.basename(p) == self.omnijar or
not self.omnijar):
jar = self._open_jar(p, f)
if 'chrome.manifest' in jar:
self.kind = 'omni'
self.omnijar = mozpath.basename(p)
self._fill_with_jar(base, jar)
continue
# If the file is a manifest, scan its entries for some referencing
# jar: urls. If there are some, the files contained in the jar they
# point to, go under a directory named after the jar.
if is_manifest(p):
m = self.files[p] if self.files.contains(p) \
else ManifestFile(base)
for e in parse_manifest(self.base, p, f.open()):
m.add(self._handle_manifest_entry(e, jars))
if self.files.contains(p):
continue
f = m
# If the file is a packed addon, unpack it under a directory named
# after the xpi.
if p.endswith('.xpi') and self._maybe_zip(f):
self._fill_with_jar(p[:-4], self._open_jar(p, f))
continue
if not p in jars:
self.files.add(p, f)
def test_bases(self):
formatter = FlatFormatter(FileRegistry())
formatter.add_base('')
formatter.add_base('browser')
formatter.add_base('webapprt')
formatter.add_base('addon0', addon=True)
self.assertEqual(formatter._get_base('platform.ini'),
('', 'platform.ini'))
self.assertEqual(formatter._get_base('browser/application.ini'),
('browser', 'application.ini'))
self.assertEqual(formatter._get_base('webapprt/webapprt.ini'),
('webapprt', 'webapprt.ini'))
self.assertEqual(formatter._get_base('addon0/install.rdf'),
('addon0', 'install.rdf'))
def is_resource(base, path):
registry = FileRegistry()
f = OmniJarFormatter(registry, 'omni.foo', non_resources=[
'defaults/messenger/mailViews.dat',
'defaults/foo/*',
'*/dummy',
])
f.add_base('')
f.add_base('app')
f.add(mozpath.join(base, path), GeneratedFile(''))
if f.copier.contains(mozpath.join(base, path)):
return False
self.assertTrue(f.copier.contains(mozpath.join(base, 'omni.foo')))
self.assertTrue(f.copier[mozpath.join(base, 'omni.foo')]
.contains(path))
return True
def describe_install_manifest(manifest, dest_dir):
try:
manifest = InstallManifest(manifest)
except UnreadableInstallManifest:
raise IOError(errno.EINVAL, 'Error parsing manifest file', manifest)
reg = FileRegistry()
mapping = {}
manifest.populate_registry(reg)
for dest_file, src in reg:
if hasattr(src, 'path'):
dest_path = mozpath.join(dest_dir, dest_file)
relsrc_path = mozpath.relpath(src.path, buildconfig.topsrcdir)
mapping[dest_path] = relsrc_path
return mapping
def test_omnijar_is_resource(self):
registry = FileRegistry()
f = OmniJarFormatter(registry,
'omni.foo',
non_resources=[
'defaults/messenger/mailViews.dat',
'defaults/foo/*',
'*/dummy',
])
f.add_base('app')
for base in ['', 'app/']:
self.assertTrue(f.is_resource(base + 'chrome'))
self.assertTrue(
f.is_resource(base + 'chrome/foo/bar/baz.properties'))
self.assertFalse(f.is_resource(base + 'chrome/icons/foo.png'))
self.assertTrue(f.is_resource(base + 'components/foo.js'))
self.assertFalse(f.is_resource(base + 'components/foo.so'))
self.assertTrue(f.is_resource(base + 'res/foo.css'))
self.assertFalse(f.is_resource(base + 'res/cursors/foo.png'))
self.assertFalse(f.is_resource(base + 'res/MainMenu.nib/'))
self.assertTrue(f.is_resource(base + 'defaults/pref/foo.js'))
self.assertFalse(
f.is_resource(base + 'defaults/pref/channel-prefs.js'))
self.assertTrue(f.is_resource(base +
'defaults/preferences/foo.js'))
self.assertFalse(
f.is_resource(base + 'defaults/preferences/channel-prefs.js'))
self.assertTrue(f.is_resource(base + 'modules/foo.jsm'))
self.assertTrue(f.is_resource(base + 'greprefs.js'))
self.assertTrue(f.is_resource(base + 'hyphenation/foo'))
self.assertTrue(f.is_resource(base + 'update.locale'))
self.assertTrue(
f.is_resource(base + 'jsloader/resource/gre/modules/foo.jsm'))
self.assertFalse(f.is_resource(base + 'foo'))
self.assertFalse(f.is_resource('foo/bar/greprefs.js'))
self.assertTrue(f.is_resource(base + 'defaults/messenger/foo.dat'))
self.assertFalse(
f.is_resource(base + 'defaults/messenger/mailViews.dat'))
self.assertTrue(f.is_resource(base + 'defaults/pref/foo.js'))
self.assertFalse(f.is_resource(base + 'defaults/foo/bar.dat'))
self.assertFalse(f.is_resource(base + 'defaults/foo/bar/baz.dat'))
self.assertTrue(f.is_resource(base + 'chrome/foo/bar/baz/dummy_'))
self.assertFalse(f.is_resource(base + 'chrome/foo/bar/baz/dummy'))
self.assertTrue(f.is_resource(base + 'chrome/foo/bar/dummy_'))
self.assertFalse(f.is_resource(base + 'chrome/foo/bar/dummy'))
def test_chrome_override(self):
registry = FileRegistry()
f = FlatFormatter(registry)
f.add_base('')
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/unix'))
# A more specific entry for a given chrome name can override a more
# generic one.
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/win', 'os=WINNT'))
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/osx',
'os=Darwin'))
# Chrome with the same name overrides the previous registration.
with self.assertRaises(ErrorMessage) as e:
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/'))
self.assertEqual(
e.exception.message, 'Error: "content foo foo/" overrides '
'"content foo foo/unix"')
# Chrome with the same name and same flags overrides the previous
# registration.
with self.assertRaises(ErrorMessage) as e:
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/',
'os=WINNT'))
self.assertEqual(
e.exception.message,
'Error: "content foo foo/ os=WINNT" overrides '
'"content foo foo/win os=WINNT"')
# We may start with the more specific entry first
f.add_manifest(ManifestContent('chrome', 'bar', 'bar/win', 'os=WINNT'))
# Then adding a more generic one overrides it.
with self.assertRaises(ErrorMessage) as e:
f.add_manifest(ManifestContent('chrome', 'bar', 'bar/unix'))
self.assertEqual(
e.exception.message, 'Error: "content bar bar/unix" overrides '
'"content bar bar/win os=WINNT"')
# Adding something more specific still works.
f.add_manifest(
ManifestContent('chrome', 'bar', 'bar/win',
'os=WINNT osversion>=7.0'))
def consume_finished(self):
mp = os.path.join(self.environment.topobjdir, '_build_manifests', 'install', '_tests')
install_manifest = InstallManifest(mp)
reg = FileRegistry()
install_manifest.populate_registry(reg)
for dest, src in reg:
if not hasattr(src, 'path'):
continue
if not os.path.isabs(dest):
dest = '_tests/' + dest
obj_path = mozpath.join(self.environment.topobjdir, dest)
if isinstance(src, PreprocessedFile):
assert os.path.exists(obj_path), '%s should exist' % obj_path
pp_info = generate_pp_info(obj_path, self.environment.topsrcdir)
else:
pp_info = None
self._install_mapping[dest] = src.path, pp_info
# Our result has four parts:
# A map from url prefixes to objdir directories:
# { "chrome://mozapps/content/": [ "dist/bin/chrome/toolkit/content/mozapps" ], ... }
# A map of overrides.
# A map from objdir paths to sourcedir paths, and an object storing mapping information for preprocessed files:
# { "dist/bin/browser/chrome/browser/content/browser/aboutSessionRestore.js":
# [ "$topsrcdir/browser/components/sessionstore/content/aboutSessionRestore.js", {} ], ... }
# An object containing build configuration information.
outputfile = os.path.join(self.environment.topobjdir, 'chrome-map.json')
with self._write_file(outputfile) as fh:
chrome_mapping = self.manifest_handler.chrome_mapping
overrides = self.manifest_handler.overrides
json.dump([
{k: list(v) for k, v in chrome_mapping.iteritems()},
overrides,
self._install_mapping,
{
'topobjdir': mozpath.normpath(self.environment.topobjdir),
'MOZ_APP_NAME': self.environment.substs.get('MOZ_APP_NAME'),
'OMNIJAR_NAME': self.environment.substs.get('OMNIJAR_NAME'),
'MOZ_MACBUNDLE_NAME': self.environment.substs.get('MOZ_MACBUNDLE_NAME'),
}
], fh, sort_keys=True, indent=2)
def is_resource(base, path):
registry = FileRegistry()
f = OmniJarFormatter(
registry,
"omni.foo",
non_resources=[
"defaults/messenger/mailViews.dat",
"defaults/foo/*",
"*/dummy",
],
)
f.add_base("")
f.add_base("app")
f.add(mozpath.join(base, path), GeneratedFile(b""))
if f.copier.contains(mozpath.join(base, path)):
return False
self.assertTrue(f.copier.contains(mozpath.join(base, "omni.foo")))
self.assertTrue(f.copier[mozpath.join(base,
"omni.foo")].contains(path))
return True
def process_manifest(destdir,
paths,
track=None,
remove_unaccounted=True,
remove_all_directory_symlinks=True,
remove_empty_directories=True,
defines={}):
if track:
if os.path.exists(track):
# We use the same format as install manifests for the tracking
# data.
manifest = InstallManifest(path=track)
remove_unaccounted = FileRegistry()
dummy_file = BaseFile()
finder = FileFinder(destdir,
find_executables=False,
find_dotfiles=True)
for dest in manifest._dests:
if '*' in dest:
for p, f in finder.find(dest):
remove_unaccounted.add(p, dummy_file)
else:
remove_unaccounted.add(dest, dummy_file)
else:
# If tracking is enabled and there is no file, we don't want to
# be removing anything.
remove_unaccounted = False
remove_empty_directories = False
remove_all_directory_symlinks = False
manifest = InstallManifest()
for path in paths:
manifest |= InstallManifest(path=path)
copier = FileCopier()
manifest.populate_registry(copier, defines_override=defines)
result = copier.copy(
destdir,
remove_unaccounted=remove_unaccounted,
remove_all_directory_symlinks=remove_all_directory_symlinks,
remove_empty_directories=remove_empty_directories)
if track:
manifest.write(path=track)
return result
def test_registry_paths(self):
self.do_test_registry_paths(FileRegistry())
def test_file_registry(self):
self.do_test_file_registry(FileRegistry())
class TestFileRegistry(BaseTestFileRegistry, unittest.TestCase):
def test_partial_paths(self):
cases = {
"foo/bar/baz/zot": ["foo/bar/baz", "foo/bar", "foo"],
"foo/bar": ["foo"],
"bar": [],
}
reg = FileRegistry()
for path, parts in six.iteritems(cases):
self.assertEqual(reg._partial_paths(path), parts)
def test_file_registry(self):
self.do_test_file_registry(FileRegistry())
def test_registry_paths(self):
self.do_test_registry_paths(FileRegistry())
def test_required_directories(self):
self.registry = FileRegistry()
self.registry.add("foo", GeneratedFile(b"foo"))
self.assertEqual(self.registry.required_directories(), set())
self.registry.add("bar/baz", GeneratedFile(b"barbaz"))
self.assertEqual(self.registry.required_directories(), {"bar"})
self.registry.add("bar/zot", GeneratedFile(b"barzot"))
self.assertEqual(self.registry.required_directories(), {"bar"})
self.registry.add("bar/zap/zot", GeneratedFile(b"barzapzot"))
self.assertEqual(self.registry.required_directories(),
{"bar", "bar/zap"})
self.registry.remove("bar/zap/zot")
self.assertEqual(self.registry.required_directories(), {"bar"})
self.registry.remove("bar/baz")
self.assertEqual(self.registry.required_directories(), {"bar"})
self.registry.remove("bar/zot")
self.assertEqual(self.registry.required_directories(), set())
self.registry.add("x/y/z", GeneratedFile(b"xyz"))
self.assertEqual(self.registry.required_directories(), {"x", "x/y"})
class TestFileRegistry(MatchTestTemplate, unittest.TestCase):
def add(self, path):
self.registry.add(path, GeneratedFile(path))
def do_check(self, pattern, result):
self.checked = True
if result:
self.assertTrue(self.registry.contains(pattern))
else:
self.assertFalse(self.registry.contains(pattern))
self.assertEqual(self.registry.match(pattern), result)
def test_file_registry(self):
self.registry = FileRegistry()
self.registry.add('foo', GeneratedFile('foo'))
bar = GeneratedFile('bar')
self.registry.add('bar', bar)
self.assertEqual(self.registry.paths(), ['foo', 'bar'])
self.assertEqual(self.registry['bar'], bar)
self.assertRaises(ErrorMessage, self.registry.add, 'foo',
GeneratedFile('foo2'))
self.assertRaises(ErrorMessage, self.registry.remove, 'qux')
self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar',
GeneratedFile('foobar'))
self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar/baz',
GeneratedFile('foobar'))
self.assertEqual(self.registry.paths(), ['foo', 'bar'])
self.registry.remove('foo')
self.assertEqual(self.registry.paths(), ['bar'])
self.registry.remove('bar')
self.assertEqual(self.registry.paths(), [])
self.prepare_match_test()
self.do_match_test()
self.assertTrue(self.checked)
self.assertEqual(self.registry.paths(), [
'bar',
'foo/bar',
'foo/baz',
'foo/qux/1',
'foo/qux/bar',
'foo/qux/2/test',
'foo/qux/2/test2',
])
self.registry.remove('foo/qux')
self.assertEqual(self.registry.paths(), ['bar', 'foo/bar', 'foo/baz'])
self.registry.add('foo/qux', GeneratedFile('fooqux'))
self.assertEqual(self.registry.paths(),
['bar', 'foo/bar', 'foo/baz', 'foo/qux'])
self.registry.remove('foo/b*')
self.assertEqual(self.registry.paths(), ['bar', 'foo/qux'])
self.assertEqual([f for f, c in self.registry], ['bar', 'foo/qux'])
self.assertEqual(len(self.registry), 2)
self.add('foo/.foo')
self.assertTrue(self.registry.contains('foo/.foo'))
class TestFileRegistry(MatchTestTemplate, unittest.TestCase):
def add(self, path):
self.registry.add(path, GeneratedFile(path))
def do_check(self, pattern, result):
self.checked = True
if result:
self.assertTrue(self.registry.contains(pattern))
else:
self.assertFalse(self.registry.contains(pattern))
self.assertEqual(self.registry.match(pattern), result)
def test_file_registry(self):
self.registry = FileRegistry()
self.registry.add('foo', GeneratedFile('foo'))
bar = GeneratedFile('bar')
self.registry.add('bar', bar)
self.assertEqual(self.registry.paths(), ['foo', 'bar'])
self.assertEqual(self.registry['bar'], bar)
self.assertRaises(ErrorMessage, self.registry.add, 'foo',
GeneratedFile('foo2'))
self.assertRaises(ErrorMessage, self.registry.remove, 'qux')
self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar',
GeneratedFile('foobar'))
self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar/baz',
GeneratedFile('foobar'))
self.assertEqual(self.registry.paths(), ['foo', 'bar'])
self.registry.remove('foo')
self.assertEqual(self.registry.paths(), ['bar'])
self.registry.remove('bar')
self.assertEqual(self.registry.paths(), [])
self.prepare_match_test()
self.do_match_test()
self.assertTrue(self.checked)
self.assertEqual(self.registry.paths(), [
'bar',
'foo/bar',
'foo/baz',
'foo/qux/1',
'foo/qux/bar',
'foo/qux/2/test',
'foo/qux/2/test2',
])
self.registry.remove('foo/qux')
self.assertEqual(self.registry.paths(), ['bar', 'foo/bar', 'foo/baz'])
self.registry.add('foo/qux', GeneratedFile('fooqux'))
self.assertEqual(self.registry.paths(), ['bar', 'foo/bar', 'foo/baz',
'foo/qux'])
self.registry.remove('foo/b*')
self.assertEqual(self.registry.paths(), ['bar', 'foo/qux'])
self.assertEqual([f for f, c in self.registry], ['bar', 'foo/qux'])
self.assertEqual(len(self.registry), 2)
self.add('foo/.foo')
self.assertTrue(self.registry.contains('foo/.foo'))
class TestFileRegistry(MatchTestTemplate, unittest.TestCase):
def add(self, path):
self.registry.add(path, GeneratedFile(path))
def do_check(self, pattern, result):
self.checked = True
if result:
self.assertTrue(self.registry.contains(pattern))
else:
self.assertFalse(self.registry.contains(pattern))
self.assertEqual(self.registry.match(pattern), result)
def test_file_registry(self):
self.registry = FileRegistry()
self.registry.add('foo', GeneratedFile('foo'))
bar = GeneratedFile('bar')
self.registry.add('bar', bar)
self.assertEqual(self.registry.paths(), ['foo', 'bar'])
self.assertEqual(self.registry['bar'], bar)
self.assertRaises(ErrorMessage, self.registry.add, 'foo',
GeneratedFile('foo2'))
self.assertRaises(ErrorMessage, self.registry.remove, 'qux')
self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar',
GeneratedFile('foobar'))
self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar/baz',
GeneratedFile('foobar'))
self.assertEqual(self.registry.paths(), ['foo', 'bar'])
self.registry.remove('foo')
self.assertEqual(self.registry.paths(), ['bar'])
self.registry.remove('bar')
self.assertEqual(self.registry.paths(), [])
self.prepare_match_test()
self.do_match_test()
self.assertTrue(self.checked)
self.assertEqual(self.registry.paths(), [
'bar',
'foo/bar',
'foo/baz',
'foo/qux/1',
'foo/qux/bar',
'foo/qux/2/test',
'foo/qux/2/test2',
])
self.registry.remove('foo/qux')
self.assertEqual(self.registry.paths(), ['bar', 'foo/bar', 'foo/baz'])
self.registry.add('foo/qux', GeneratedFile('fooqux'))
self.assertEqual(self.registry.paths(),
['bar', 'foo/bar', 'foo/baz', 'foo/qux'])
self.registry.remove('foo/b*')
self.assertEqual(self.registry.paths(), ['bar', 'foo/qux'])
self.assertEqual([f for f, c in self.registry], ['bar', 'foo/qux'])
self.assertEqual(len(self.registry), 2)
self.add('foo/.foo')
self.assertTrue(self.registry.contains('foo/.foo'))
def test_registry_paths(self):
self.registry = FileRegistry()
# Can't add a file if it requires a directory in place of a
# file we also require.
self.registry.add('foo', GeneratedFile('foo'))
self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar',
GeneratedFile('foobar'))
# Can't add a file if we already have a directory there.
self.registry.add('bar/baz', GeneratedFile('barbaz'))
self.assertRaises(ErrorMessage, self.registry.add, 'bar',
GeneratedFile('bar'))
# Bump the count of things that require bar/ to 2.
self.registry.add('bar/zot', GeneratedFile('barzot'))
self.assertRaises(ErrorMessage, self.registry.add, 'bar',
GeneratedFile('bar'))
# Drop the count of things that require bar/ to 1.
self.registry.remove('bar/baz')
self.assertRaises(ErrorMessage, self.registry.add, 'bar',
GeneratedFile('bar'))
# Drop the count of things that require bar/ to 0.
self.registry.remove('bar/zot')
self.registry.add('bar/zot', GeneratedFile('barzot'))
def test_required_directories(self):
self.registry = FileRegistry()
self.registry.add('foo', GeneratedFile('foo'))
self.assertEqual(self.registry.required_directories(), set())
self.registry.add('bar/baz', GeneratedFile('barbaz'))
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.add('bar/zot', GeneratedFile('barzot'))
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.add('bar/zap/zot', GeneratedFile('barzapzot'))
self.assertEqual(self.registry.required_directories(),
{'bar', 'bar/zap'})
self.registry.remove('bar/zap/zot')
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.remove('bar/baz')
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.remove('bar/zot')
self.assertEqual(self.registry.required_directories(), set())
self.registry.add('x/y/z', GeneratedFile('xyz'))
self.assertEqual(self.registry.required_directories(), {'x', 'x/y'})
class UnpackFinder(BaseFinder):
"""
Special Finder object that treats the source package directory as if it
were in the flat chrome format, whatever chrome format it actually is in.
This means that for example, paths like chrome/browser/content/... match
files under jar:chrome/browser.jar!/content/... in case of jar chrome
format.
The only argument to the constructor is a Finder instance or a path.
The UnpackFinder is populated with files from this Finder instance,
or with files from a FileFinder using the given path as its root.
"""
def __init__(self, source, omnijar_name=None, unpack_xpi=True):
if isinstance(source, BaseFinder):
self._finder = source
else:
self._finder = FileFinder(source)
self.base = self._finder.base
self.files = FileRegistry()
self.kind = "flat"
if omnijar_name:
self.omnijar = omnijar_name
else:
# Can't include globally because of bootstrapping issues.
from buildconfig import substs
self.omnijar = substs.get("OMNIJAR_NAME", "omni.ja")
self.jarlogs = {}
self.compressed = False
self._unpack_xpi = unpack_xpi
jars = set()
for p, f in self._finder.find("*"):
# Skip the precomplete file, which is generated at packaging time.
if p == "precomplete":
continue
base = mozpath.dirname(p)
# If the file matches the omnijar pattern, it is an omnijar.
# All the files it contains go in the directory containing the full
# pattern. Manifests are merged if there is a corresponding manifest
# in the directory.
if self._maybe_zip(f) and mozpath.match(p, "**/%s" % self.omnijar):
jar = self._open_jar(p, f)
if "chrome.manifest" in jar:
self.kind = "omni"
self._fill_with_jar(p[: -len(self.omnijar) - 1], jar)
continue
# If the file is a manifest, scan its entries for some referencing
# jar: urls. If there are some, the files contained in the jar they
# point to, go under a directory named after the jar.
if is_manifest(p):
m = self.files[p] if self.files.contains(p) else ManifestFile(base)
for e in parse_manifest(
self.base, p, codecs.getreader("utf-8")(f.open())
):
m.add(self._handle_manifest_entry(e, jars))
if self.files.contains(p):
continue
f = m
# If we're unpacking packed addons and the file is a packed addon,
# unpack it under a directory named after the xpi.
if self._unpack_xpi and p.endswith(".xpi") and self._maybe_zip(f):
self._fill_with_jar(p[:-4], self._open_jar(p, f))
continue
if p not in jars:
self.files.add(p, f)
def _fill_with_jar(self, base, jar):
for j in jar:
path = mozpath.join(base, j.filename)
if is_manifest(j.filename):
m = (
self.files[path]
if self.files.contains(path)
else ManifestFile(mozpath.dirname(path))
)
for e in parse_manifest(None, path, j):
m.add(e)
if not self.files.contains(path):
self.files.add(path, m)
continue
else:
self.files.add(path, DeflatedFile(j))
def _handle_manifest_entry(self, entry, jars):
jarpath = None
if (
isinstance(entry, ManifestEntryWithRelPath)
and urlparse(entry.relpath).scheme == "jar"
):
jarpath, entry = self._unjarize(entry, entry.relpath)
elif (
isinstance(entry, ManifestResource)
and urlparse(entry.target).scheme == "jar"
):
jarpath, entry = self._unjarize(entry, entry.target)
if jarpath:
# Don't defer unpacking the jar file. If we already saw
# it, take (and remove) it from the registry. If we
# haven't, try to find it now.
if self.files.contains(jarpath):
jar = self.files[jarpath]
self.files.remove(jarpath)
else:
jar = [f for p, f in self._finder.find(jarpath)]
assert len(jar) == 1
jar = jar[0]
if jarpath not in jars:
base = mozpath.splitext(jarpath)[0]
for j in self._open_jar(jarpath, jar):
self.files.add(mozpath.join(base, j.filename), DeflatedFile(j))
jars.add(jarpath)
self.kind = "jar"
return entry
def _open_jar(self, path, file):
"""
Return a JarReader for the given BaseFile instance, keeping a log of
the preloaded entries it has.
"""
jar = JarReader(fileobj=file.open())
self.compressed = max(self.compressed, jar.compression)
if jar.last_preloaded:
jarlog = list(jar.entries.keys())
self.jarlogs[path] = jarlog[: jarlog.index(jar.last_preloaded) + 1]
return jar
def find(self, path):
for p in self.files.match(path):
yield p, self.files[p]
def _maybe_zip(self, file):
"""
Return whether the given BaseFile looks like a ZIP/Jar.
"""
header = file.open().read(8)
return len(header) == 8 and (header[0:2] == b"PK" or header[4:6] == b"PK")
def _unjarize(self, entry, relpath):
"""
Transform a manifest entry pointing to chrome data in a jar in one
pointing to the corresponding unpacked path. Return the jar path and
the new entry.
"""
base = entry.base
jar, relpath = urlparse(relpath).path.split("!", 1)
entry = (
entry.rebase(mozpath.join(base, "jar:%s!" % jar))
.move(mozpath.join(base, mozpath.splitext(jar)[0]))
.rebase(base)
)
return mozpath.join(base, jar), entry
def test_registry_paths_subtree(self):
FileRegistry()
self.do_test_registry_paths(self.create_registry())
def test_file_registry_subtree_base(self):
registry = FileRegistry()
self.assertEqual(registry, FileRegistrySubtree("", registry))
self.assertNotEqual(registry, FileRegistrySubtree("base", registry))
class TestFileRegistry(BaseTestFileRegistry, unittest.TestCase):
def test_partial_paths(self):
cases = {"foo/bar/baz/zot": ["foo/bar/baz", "foo/bar", "foo"], "foo/bar": ["foo"], "bar": []}
reg = FileRegistry()
for path, parts in cases.iteritems():
self.assertEqual(reg._partial_paths(path), parts)
def test_file_registry(self):
self.do_test_file_registry(FileRegistry())
def test_registry_paths(self):
self.do_test_registry_paths(FileRegistry())
def test_required_directories(self):
self.registry = FileRegistry()
self.registry.add("foo", GeneratedFile("foo"))
self.assertEqual(self.registry.required_directories(), set())
self.registry.add("bar/baz", GeneratedFile("barbaz"))
self.assertEqual(self.registry.required_directories(), {"bar"})
self.registry.add("bar/zot", GeneratedFile("barzot"))
self.assertEqual(self.registry.required_directories(), {"bar"})
self.registry.add("bar/zap/zot", GeneratedFile("barzapzot"))
self.assertEqual(self.registry.required_directories(), {"bar", "bar/zap"})
self.registry.remove("bar/zap/zot")
self.assertEqual(self.registry.required_directories(), {"bar"})
self.registry.remove("bar/baz")
self.assertEqual(self.registry.required_directories(), {"bar"})
self.registry.remove("bar/zot")
self.assertEqual(self.registry.required_directories(), set())
self.registry.add("x/y/z", GeneratedFile("xyz"))
self.assertEqual(self.registry.required_directories(), {"x", "x/y"})
class TestFileRegistry(MatchTestTemplate, unittest.TestCase):
def add(self, path):
self.registry.add(path, GeneratedFile(path))
def do_check(self, pattern, result):
self.checked = True
if result:
self.assertTrue(self.registry.contains(pattern))
else:
self.assertFalse(self.registry.contains(pattern))
self.assertEqual(self.registry.match(pattern), result)
def test_file_registry(self):
self.registry = FileRegistry()
self.registry.add('foo', GeneratedFile('foo'))
bar = GeneratedFile('bar')
self.registry.add('bar', bar)
self.assertEqual(self.registry.paths(), ['foo', 'bar'])
self.assertEqual(self.registry['bar'], bar)
self.assertRaises(ErrorMessage, self.registry.add, 'foo',
GeneratedFile('foo2'))
self.assertRaises(ErrorMessage, self.registry.remove, 'qux')
self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar',
GeneratedFile('foobar'))
self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar/baz',
GeneratedFile('foobar'))
self.assertEqual(self.registry.paths(), ['foo', 'bar'])
self.registry.remove('foo')
self.assertEqual(self.registry.paths(), ['bar'])
self.registry.remove('bar')
self.assertEqual(self.registry.paths(), [])
self.prepare_match_test()
self.do_match_test()
self.assertTrue(self.checked)
self.assertEqual(self.registry.paths(), [
'bar',
'foo/bar',
'foo/baz',
'foo/qux/1',
'foo/qux/bar',
'foo/qux/2/test',
'foo/qux/2/test2',
])
self.registry.remove('foo/qux')
self.assertEqual(self.registry.paths(), ['bar', 'foo/bar', 'foo/baz'])
self.registry.add('foo/qux', GeneratedFile('fooqux'))
self.assertEqual(self.registry.paths(), ['bar', 'foo/bar', 'foo/baz',
'foo/qux'])
self.registry.remove('foo/b*')
self.assertEqual(self.registry.paths(), ['bar', 'foo/qux'])
self.assertEqual([f for f, c in self.registry], ['bar', 'foo/qux'])
self.assertEqual(len(self.registry), 2)
self.add('foo/.foo')
self.assertTrue(self.registry.contains('foo/.foo'))
def test_registry_paths(self):
self.registry = FileRegistry()
# Can't add a file if it requires a directory in place of a
# file we also require.
self.registry.add('foo', GeneratedFile('foo'))
self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar',
GeneratedFile('foobar'))
# Can't add a file if we already have a directory there.
self.registry.add('bar/baz', GeneratedFile('barbaz'))
self.assertRaises(ErrorMessage, self.registry.add, 'bar',
GeneratedFile('bar'))
# Bump the count of things that require bar/ to 2.
self.registry.add('bar/zot', GeneratedFile('barzot'))
self.assertRaises(ErrorMessage, self.registry.add, 'bar',
GeneratedFile('bar'))
# Drop the count of things that require bar/ to 1.
self.registry.remove('bar/baz')
self.assertRaises(ErrorMessage, self.registry.add, 'bar',
GeneratedFile('bar'))
# Drop the count of things that require bar/ to 0.
self.registry.remove('bar/zot')
self.registry.add('bar/zot', GeneratedFile('barzot'))
def test_required_directories(self):
self.registry = FileRegistry()
self.registry.add('foo', GeneratedFile('foo'))
self.assertEqual(self.registry.required_directories(), set())
self.registry.add('bar/baz', GeneratedFile('barbaz'))
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.add('bar/zot', GeneratedFile('barzot'))
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.add('bar/zap/zot', GeneratedFile('barzapzot'))
self.assertEqual(self.registry.required_directories(), {'bar', 'bar/zap'})
self.registry.remove('bar/zap/zot')
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.remove('bar/baz')
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.remove('bar/zot')
self.assertEqual(self.registry.required_directories(), set())
self.registry.add('x/y/z', GeneratedFile('xyz'))
self.assertEqual(self.registry.required_directories(), {'x', 'x/y'})
def test_omnijar_formatter(self):
registry = FileRegistry()
formatter = OmniJarFormatter(registry, 'omni.foo')
fill_formatter(formatter, CONTENTS)
RESULT = {
'omni.foo': {
'chrome.manifest': [
'manifest chrome/chrome.manifest',
'manifest components/components.manifest',
],
'chrome/chrome.manifest': [
'manifest f/f.manifest',
],
'chrome/f/f.manifest': [
'content oo oo/',
'content bar oo/bar/',
'resource foo resource://bar/',
],
'chrome/f/oo/bar/baz':
CONTENTS['chrome/f/oo/bar/baz'],
'chrome/f/oo/baz':
CONTENTS['chrome/f/oo/baz'],
'chrome/f/oo/qux':
CONTENTS['chrome/f/oo/qux'],
'components/components.manifest': [
'interfaces interfaces.xpt',
],
'components/interfaces.xpt': {
'foo': read_interfaces(foo_xpt.open())['foo'],
'bar': read_interfaces(bar_xpt.open())['bar'],
},
},
'chrome.manifest': [
'manifest components/components.manifest',
],
'components/components.manifest': [
'binary-component foo.so',
],
'components/foo.so':
CONTENTS['components/foo.so'],
'foo':
CONTENTS['foo'],
'app/omni.foo': {
'chrome.manifest': [
'manifest chrome/chrome.manifest',
'manifest components/components.manifest',
],
'chrome/chrome.manifest': [
'content content foo/',
],
'chrome/foo/foo':
CONTENTS['app/chrome/foo/foo'],
'components/components.manifest': [
'component {foo-id} foo.js',
],
'components/foo.js':
CONTENTS['app/components/foo.js'],
},
'addon0/chrome.manifest': [
'manifest chrome/chrome.manifest',
'manifest components/components.manifest',
],
'addon0/chrome/chrome.manifest': [
'content content jar:foo.jar!/bar/',
],
'addon0/chrome/foo.jar': {
'bar/baz': CONTENTS['addon0/chrome/foo/bar/baz'],
},
'addon0/components/components.manifest': [
'interfaces interfaces.xpt',
],
'addon0/components/interfaces.xpt': {
'foo': read_interfaces(foo2_xpt.open())['foo'],
'bar': read_interfaces(bar_xpt.open())['bar'],
},
}
self.assertEqual(get_contents(registry), RESULT)
def create_registry(self):
registry = FileRegistry()
registry.add("foo/bar", GeneratedFile(b"foo/bar"))
registry.add("baz/qux", GeneratedFile(b"baz/qux"))
return FileRegistrySubtree("base/root", registry)
class UnpackFinder(FileFinder):
'''
Special FileFinder that treats the source package directory as if it were
in the flat chrome format, whatever chrome format it actually is in.
This means that for example, paths like chrome/browser/content/... match
files under jar:chrome/browser.jar!/content/... in case of jar chrome
format.
'''
def __init__(self, *args, **kargs):
FileFinder.__init__(self, *args, **kargs)
self.files = FileRegistry()
self.kind = 'flat'
self.omnijar = None
self.jarlogs = {}
self.optimizedjars = False
jars = set()
for p, f in FileFinder.find(self, '*'):
# Skip the precomplete file, which is generated at packaging time.
if p == 'precomplete':
continue
base = mozpack.path.dirname(p)
# If the file is a zip/jar that is not a .xpi, and contains a
# chrome.manifest, it is an omnijar. All the files it contains
# go in the directory containing the omnijar. Manifests are merged
# if there is a corresponding manifest in the directory.
if not p.endswith('.xpi') and self._maybe_zip(f) and \
(mozpack.path.basename(p) == self.omnijar or
not self.omnijar):
jar = self._open_jar(p, f)
if 'chrome.manifest' in jar:
self.kind = 'omni'
self.omnijar = mozpack.path.basename(p)
self._fill_with_omnijar(base, jar)
continue
# If the file is a manifest, scan its entries for some referencing
# jar: urls. If there are some, the files contained in the jar they
# point to, go under a directory named after the jar.
if is_manifest(p):
m = self.files[p] if self.files.contains(p) \
else ManifestFile(base)
for e in parse_manifest(self.base, p, f.open()):
m.add(self._handle_manifest_entry(e, jars))
if self.files.contains(p):
continue
f = m
if not p in jars:
self.files.add(p, f)
def _fill_with_omnijar(self, base, jar):
for j in jar:
path = mozpack.path.join(base, j.filename)
if is_manifest(j.filename):
m = self.files[path] if self.files.contains(path) \
else ManifestFile(mozpack.path.dirname(path))
for e in parse_manifest(None, path, j):
m.add(e)
if not self.files.contains(path):
self.files.add(path, m)
continue
else:
self.files.add(path, DeflatedFile(j))
def _handle_manifest_entry(self, entry, jars):
jarpath = None
if isinstance(entry, ManifestEntryWithRelPath) and \
urlparse(entry.relpath).scheme == 'jar':
jarpath, entry = self._unjarize(entry, entry.relpath)
elif isinstance(entry, ManifestResource) and \
urlparse(entry.target).scheme == 'jar':
jarpath, entry = self._unjarize(entry, entry.target)
if jarpath:
# Don't defer unpacking the jar file. If we already saw
# it, take (and remove) it from the registry. If we
# haven't, try to find it now.
if self.files.contains(jarpath):
jar = self.files[jarpath]
self.files.remove(jarpath)
else:
jar = [f for p, f in FileFinder.find(self, jarpath)]
assert len(jar) == 1
jar = jar[0]
if not jarpath in jars:
base = mozpack.path.splitext(jarpath)[0]
for j in self._open_jar(jarpath, jar):
self.files.add(mozpack.path.join(base, j.filename),
DeflatedFile(j))
jars.add(jarpath)
self.kind = 'jar'
return entry
def _open_jar(self, path, file):
'''
Return a JarReader for the given BaseFile instance, keeping a log of
the preloaded entries it has.
'''
jar = JarReader(fileobj=file.open())
if jar.is_optimized:
self.optimizedjars = True
if jar.last_preloaded:
jarlog = jar.entries.keys()
self.jarlogs[path] = jarlog[:jarlog.index(jar.last_preloaded) + 1]
return jar
def find(self, path):
for p in self.files.match(path):
yield p, self.files[p]
def _maybe_zip(self, file):
'''
Return whether the given BaseFile looks like a ZIP/Jar.
'''
header = file.open().read(8)
return len(header) == 8 and (header[0:2] == 'PK'
or header[4:6] == 'PK')
def _unjarize(self, entry, relpath):
'''
Transform a manifest entry pointing to chrome data in a jar in one
pointing to the corresponding unpacked path. Return the jar path and
the new entry.
'''
base = entry.base
jar, relpath = urlparse(relpath).path.split('!', 1)
entry = entry.rebase(mozpack.path.join(base, 'jar:%s!' % jar)) \
.move(mozpack.path.join(base, mozpack.path.splitext(jar)[0])) \
.rebase(base)
return mozpack.path.join(base, jar), entry
def test_l10n_repack(self):
foo = GeneratedFile('foo')
foobar = GeneratedFile('foobar')
qux = GeneratedFile('qux')
bar = GeneratedFile('bar')
baz = GeneratedFile('baz')
dict_aa = GeneratedFile('dict_aa')
dict_bb = GeneratedFile('dict_bb')
dict_cc = GeneratedFile('dict_cc')
barbaz = GeneratedFile('barbaz')
lst = GeneratedFile('foo\nbar')
app_finder = MockFinder({
'bar/foo':
foo,
'chrome/foo/foobar':
foobar,
'chrome/qux/qux.properties':
qux,
'chrome/qux/baz/baz.properties':
baz,
'chrome/chrome.manifest':
ManifestFile('chrome', [
ManifestContent('chrome', 'foo', 'foo/'),
ManifestLocale('chrome', 'qux', 'en-US', 'qux/'),
]),
'chrome.manifest':
ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]),
'dict/aa':
dict_aa,
'app/chrome/bar/barbaz.dtd':
barbaz,
'app/chrome/chrome.manifest':
ManifestFile(
'app/chrome',
[ManifestLocale('app/chrome', 'bar', 'en-US', 'bar/')]),
'app/chrome.manifest':
ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]),
'app/dict/bb':
dict_bb,
'app/dict/cc':
dict_cc,
'app/chrome/bar/search/foo.xml':
foo,
'app/chrome/bar/search/bar.xml':
bar,
'app/chrome/bar/search/lst.txt':
lst,
})
app_finder.jarlogs = {}
app_finder.base = 'app'
foo_l10n = GeneratedFile('foo_l10n')
qux_l10n = GeneratedFile('qux_l10n')
baz_l10n = GeneratedFile('baz_l10n')
barbaz_l10n = GeneratedFile('barbaz_l10n')
lst_l10n = GeneratedFile('foo\nqux')
l10n_finder = MockFinder({
'chrome/qux-l10n/qux.properties':
qux_l10n,
'chrome/qux-l10n/baz/baz.properties':
baz_l10n,
'chrome/chrome.manifest':
ManifestFile('chrome', [
ManifestLocale('chrome', 'qux', 'x-test', 'qux-l10n/'),
]),
'chrome.manifest':
ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]),
'dict/bb':
dict_bb,
'dict/cc':
dict_cc,
'app/chrome/bar-l10n/barbaz.dtd':
barbaz_l10n,
'app/chrome/chrome.manifest':
ManifestFile(
'app/chrome',
[ManifestLocale('app/chrome', 'bar', 'x-test', 'bar-l10n/')]),
'app/chrome.manifest':
ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]),
'app/dict/aa':
dict_aa,
'app/chrome/bar-l10n/search/foo.xml':
foo_l10n,
'app/chrome/bar-l10n/search/qux.xml':
qux_l10n,
'app/chrome/bar-l10n/search/lst.txt':
lst_l10n,
})
l10n_finder.base = 'l10n'
copier = FileRegistry()
formatter = FlatFormatter(copier)
l10n._repack(app_finder, l10n_finder, copier, formatter,
['dict', 'chrome/**/search/*.xml'])
self.maxDiff = None
repacked = {
'bar/foo':
foo,
'chrome/foo/foobar':
foobar,
'chrome/qux-l10n/qux.properties':
qux_l10n,
'chrome/qux-l10n/baz/baz.properties':
baz_l10n,
'chrome/chrome.manifest':
ManifestFile('chrome', [
ManifestContent('chrome', 'foo', 'foo/'),
ManifestLocale('chrome', 'qux', 'x-test', 'qux-l10n/'),
]),
'chrome.manifest':
ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]),
'dict/bb':
dict_bb,
'dict/cc':
dict_cc,
'app/chrome/bar-l10n/barbaz.dtd':
barbaz_l10n,
'app/chrome/chrome.manifest':
ManifestFile(
'app/chrome',
[ManifestLocale('app/chrome', 'bar', 'x-test', 'bar-l10n/')]),
'app/chrome.manifest':
ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]),
'app/dict/aa':
dict_aa,
'app/chrome/bar-l10n/search/foo.xml':
foo_l10n,
'app/chrome/bar-l10n/search/qux.xml':
qux_l10n,
'app/chrome/bar-l10n/search/lst.txt':
lst_l10n,
}
self.assertEqual(
dict((p, f.open().read()) for p, f in copier),
dict((p, f.open().read()) for p, f in repacked.iteritems()))
class TestFileRegistry(BaseTestFileRegistry, unittest.TestCase):
def test_partial_paths(self):
cases = {
'foo/bar/baz/zot': ['foo/bar/baz', 'foo/bar', 'foo'],
'foo/bar': ['foo'],
'bar': [],
}
reg = FileRegistry()
for path, parts in cases.iteritems():
self.assertEqual(reg._partial_paths(path), parts)
def test_file_registry(self):
self.do_test_file_registry(FileRegistry())
def test_registry_paths(self):
self.do_test_registry_paths(FileRegistry())
def test_required_directories(self):
self.registry = FileRegistry()
self.registry.add('foo', GeneratedFile('foo'))
self.assertEqual(self.registry.required_directories(), set())
self.registry.add('bar/baz', GeneratedFile('barbaz'))
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.add('bar/zot', GeneratedFile('barzot'))
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.add('bar/zap/zot', GeneratedFile('barzapzot'))
self.assertEqual(self.registry.required_directories(), {'bar', 'bar/zap'})
self.registry.remove('bar/zap/zot')
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.remove('bar/baz')
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.remove('bar/zot')
self.assertEqual(self.registry.required_directories(), set())
self.registry.add('x/y/z', GeneratedFile('xyz'))
self.assertEqual(self.registry.required_directories(), {'x', 'x/y'})
def __init__(self, interfaces):
from mozpack.copier import FileRegistry
self.interfaces = interfaces
self.registry = FileRegistry()
class TestFileRegistry(BaseTestFileRegistry, unittest.TestCase):
def test_partial_paths(self):
cases = {
'foo/bar/baz/zot': ['foo/bar/baz', 'foo/bar', 'foo'],
'foo/bar': ['foo'],
'bar': [],
}
reg = FileRegistry()
for path, parts in cases.iteritems():
self.assertEqual(reg._partial_paths(path), parts)
def test_file_registry(self):
self.do_test_file_registry(FileRegistry())
def test_registry_paths(self):
self.do_test_registry_paths(FileRegistry())
def test_required_directories(self):
self.registry = FileRegistry()
self.registry.add('foo', GeneratedFile('foo'))
self.assertEqual(self.registry.required_directories(), set())
self.registry.add('bar/baz', GeneratedFile('barbaz'))
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.add('bar/zot', GeneratedFile('barzot'))
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.add('bar/zap/zot', GeneratedFile('barzapzot'))
self.assertEqual(self.registry.required_directories(),
{'bar', 'bar/zap'})
self.registry.remove('bar/zap/zot')
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.remove('bar/baz')
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.remove('bar/zot')
self.assertEqual(self.registry.required_directories(), set())
self.registry.add('x/y/z', GeneratedFile('xyz'))
self.assertEqual(self.registry.required_directories(), {'x', 'x/y'})
def __init__(self, interfaces):
from mozpack.copier import FileRegistry
self.interfaces = interfaces;
self.registry = FileRegistry()
def create_registry(self):
registry = FileRegistry()
registry.add('foo/bar', GeneratedFile('foo/bar'))
registry.add('baz/qux', GeneratedFile('baz/qux'))
return FileRegistrySubtree('base/root', registry)
def create_registry(self):
registry = FileRegistry()
registry.add("foo/bar", GeneratedFile("foo/bar"))
registry.add("baz/qux", GeneratedFile("baz/qux"))
return FileRegistrySubtree("base/root", registry)
class UnpackFinder(FileFinder):
'''
Special FileFinder that treats the source package directory as if it were
in the flat chrome format, whatever chrome format it actually is in.
This means that for example, paths like chrome/browser/content/... match
files under jar:chrome/browser.jar!/content/... in case of jar chrome
format.
'''
def __init__(self, *args, **kargs):
FileFinder.__init__(self, *args, **kargs)
self.files = FileRegistry()
self.kind = 'flat'
self.omnijar = None
self.jarlogs = {}
self.optimizedjars = False
jars = set()
for p, f in FileFinder.find(self, '*'):
# Skip the precomplete file, which is generated at packaging time.
if p == 'precomplete':
continue
base = mozpack.path.dirname(p)
# If the file is a zip/jar that is not a .xpi, and contains a
# chrome.manifest, it is an omnijar. All the files it contains
# go in the directory containing the omnijar. Manifests are merged
# if there is a corresponding manifest in the directory.
if not p.endswith('.xpi') and self._maybe_zip(f) and \
(mozpack.path.basename(p) == self.omnijar or
not self.omnijar):
jar = self._open_jar(p, f)
if 'chrome.manifest' in jar:
self.kind = 'omni'
self.omnijar = mozpack.path.basename(p)
self._fill_with_omnijar(base, jar)
continue
# If the file is a manifest, scan its entries for some referencing
# jar: urls. If there are some, the files contained in the jar they
# point to, go under a directory named after the jar.
if is_manifest(p):
m = self.files[p] if self.files.contains(p) \
else ManifestFile(base)
for e in parse_manifest(self.base, p, f.open()):
m.add(self._handle_manifest_entry(e, jars))
if self.files.contains(p):
continue
f = m
if not p in jars:
self.files.add(p, f)
def _fill_with_omnijar(self, base, jar):
for j in jar:
path = mozpack.path.join(base, j.filename)
if is_manifest(j.filename):
m = self.files[path] if self.files.contains(path) \
else ManifestFile(mozpack.path.dirname(path))
for e in parse_manifest(None, path, j):
m.add(e)
if not self.files.contains(path):
self.files.add(path, m)
continue
else:
self.files.add(path, DeflatedFile(j))
def _handle_manifest_entry(self, entry, jars):
jarpath = None
if isinstance(entry, ManifestEntryWithRelPath) and \
urlparse(entry.relpath).scheme == 'jar':
jarpath, entry = self._unjarize(entry, entry.relpath)
elif isinstance(entry, ManifestResource) and \
urlparse(entry.target).scheme == 'jar':
jarpath, entry = self._unjarize(entry, entry.target)
if jarpath:
# Don't defer unpacking the jar file. If we already saw
# it, take (and remove) it from the registry. If we
# haven't, try to find it now.
if self.files.contains(jarpath):
jar = self.files[jarpath]
self.files.remove(jarpath)
else:
jar = [f for p, f in FileFinder.find(self, jarpath)]
assert len(jar) == 1
jar = jar[0]
if not jarpath in jars:
base = mozpack.path.splitext(jarpath)[0]
for j in self._open_jar(jarpath, jar):
self.files.add(mozpack.path.join(base,
j.filename),
DeflatedFile(j))
jars.add(jarpath)
self.kind = 'jar'
return entry
def _open_jar(self, path, file):
'''
Return a JarReader for the given BaseFile instance, keeping a log of
the preloaded entries it has.
'''
jar = JarReader(fileobj=file.open())
if jar.is_optimized:
self.optimizedjars = True
if jar.last_preloaded:
jarlog = jar.entries.keys()
self.jarlogs[path] = jarlog[:jarlog.index(jar.last_preloaded) + 1]
return jar
def find(self, path):
for p in self.files.match(path):
yield p, self.files[p]
def _maybe_zip(self, file):
'''
Return whether the given BaseFile looks like a ZIP/Jar.
'''
header = file.open().read(8)
return len(header) == 8 and (header[0:2] == 'PK' or
header[4:6] == 'PK')
def _unjarize(self, entry, relpath):
'''
Transform a manifest entry pointing to chrome data in a jar in one
pointing to the corresponding unpacked path. Return the jar path and
the new entry.
'''
base = entry.base
jar, relpath = urlparse(relpath).path.split('!', 1)
entry = entry.rebase(mozpack.path.join(base, 'jar:%s!' % jar)) \
.move(mozpack.path.join(base, mozpack.path.splitext(jar)[0])) \
.rebase(base)
return mozpack.path.join(base, jar), entry
def test_partial_paths(self):
cases = {"foo/bar/baz/zot": ["foo/bar/baz", "foo/bar", "foo"], "foo/bar": ["foo"], "bar": []}
reg = FileRegistry()
for path, parts in cases.iteritems():
self.assertEqual(reg._partial_paths(path), parts)
def create_registry(self):
registry = FileRegistry()
registry.add('foo/bar', GeneratedFile('foo/bar'))
registry.add('baz/qux', GeneratedFile('baz/qux'))
return FileRegistrySubtree('base/root', registry)
