class ComposedFinder(BaseFinder): ''' Composes multiple File Finders in some sort of virtual file system. A ComposedFinder is initialized from a dictionary associating paths to *Finder instances. Note this could be optimized to be smarter than getting all the files in advance. ''' def __init__(self, finders): # Can't import globally, because of the dependency of mozpack.copier # on this module. from mozpack.copier import FileRegistry self.files = FileRegistry() for base, finder in sorted(finders.iteritems()): if self.files.contains(base): self.files.remove(base) for p, f in finder.find(''): self.files.add(mozpath.join(base, p), f) def find(self, pattern): for p in self.files.match(pattern): yield p, self.files[p]
class TestFileRegistry(BaseTestFileRegistry, unittest.TestCase): def test_partial_paths(self): cases = { "foo/bar/baz/zot": ["foo/bar/baz", "foo/bar", "foo"], "foo/bar": ["foo"], "bar": [], } reg = FileRegistry() for path, parts in six.iteritems(cases): self.assertEqual(reg._partial_paths(path), parts) def test_file_registry(self): self.do_test_file_registry(FileRegistry()) def test_registry_paths(self): self.do_test_registry_paths(FileRegistry()) def test_required_directories(self): self.registry = FileRegistry() self.registry.add("foo", GeneratedFile(b"foo")) self.assertEqual(self.registry.required_directories(), set()) self.registry.add("bar/baz", GeneratedFile(b"barbaz")) self.assertEqual(self.registry.required_directories(), {"bar"}) self.registry.add("bar/zot", GeneratedFile(b"barzot")) self.assertEqual(self.registry.required_directories(), {"bar"}) self.registry.add("bar/zap/zot", GeneratedFile(b"barzapzot")) self.assertEqual(self.registry.required_directories(), {"bar", "bar/zap"}) self.registry.remove("bar/zap/zot") self.assertEqual(self.registry.required_directories(), {"bar"}) self.registry.remove("bar/baz") self.assertEqual(self.registry.required_directories(), {"bar"}) self.registry.remove("bar/zot") self.assertEqual(self.registry.required_directories(), set()) self.registry.add("x/y/z", GeneratedFile(b"xyz")) self.assertEqual(self.registry.required_directories(), {"x", "x/y"})
class TestFileRegistry(BaseTestFileRegistry, unittest.TestCase): def test_partial_paths(self): cases = { 'foo/bar/baz/zot': ['foo/bar/baz', 'foo/bar', 'foo'], 'foo/bar': ['foo'], 'bar': [], } reg = FileRegistry() for path, parts in cases.iteritems(): self.assertEqual(reg._partial_paths(path), parts) def test_file_registry(self): self.do_test_file_registry(FileRegistry()) def test_registry_paths(self): self.do_test_registry_paths(FileRegistry()) def test_required_directories(self): self.registry = FileRegistry() self.registry.add('foo', GeneratedFile('foo')) self.assertEqual(self.registry.required_directories(), set()) self.registry.add('bar/baz', GeneratedFile('barbaz')) self.assertEqual(self.registry.required_directories(), {'bar'}) self.registry.add('bar/zot', GeneratedFile('barzot')) self.assertEqual(self.registry.required_directories(), {'bar'}) self.registry.add('bar/zap/zot', GeneratedFile('barzapzot')) self.assertEqual(self.registry.required_directories(), {'bar', 'bar/zap'}) self.registry.remove('bar/zap/zot') self.assertEqual(self.registry.required_directories(), {'bar'}) self.registry.remove('bar/baz') self.assertEqual(self.registry.required_directories(), {'bar'}) self.registry.remove('bar/zot') self.assertEqual(self.registry.required_directories(), set()) self.registry.add('x/y/z', GeneratedFile('xyz')) self.assertEqual(self.registry.required_directories(), {'x', 'x/y'})
class TestFileRegistry(BaseTestFileRegistry, unittest.TestCase): def test_partial_paths(self): cases = {"foo/bar/baz/zot": ["foo/bar/baz", "foo/bar", "foo"], "foo/bar": ["foo"], "bar": []} reg = FileRegistry() for path, parts in cases.iteritems(): self.assertEqual(reg._partial_paths(path), parts) def test_file_registry(self): self.do_test_file_registry(FileRegistry()) def test_registry_paths(self): self.do_test_registry_paths(FileRegistry()) def test_required_directories(self): self.registry = FileRegistry() self.registry.add("foo", GeneratedFile("foo")) self.assertEqual(self.registry.required_directories(), set()) self.registry.add("bar/baz", GeneratedFile("barbaz")) self.assertEqual(self.registry.required_directories(), {"bar"}) self.registry.add("bar/zot", GeneratedFile("barzot")) self.assertEqual(self.registry.required_directories(), {"bar"}) self.registry.add("bar/zap/zot", GeneratedFile("barzapzot")) self.assertEqual(self.registry.required_directories(), {"bar", "bar/zap"}) self.registry.remove("bar/zap/zot") self.assertEqual(self.registry.required_directories(), {"bar"}) self.registry.remove("bar/baz") self.assertEqual(self.registry.required_directories(), {"bar"}) self.registry.remove("bar/zot") self.assertEqual(self.registry.required_directories(), set()) self.registry.add("x/y/z", GeneratedFile("xyz")) self.assertEqual(self.registry.required_directories(), {"x", "x/y"})
class UnpackFinder(FileFinder): ''' Special FileFinder that treats the source package directory as if it were in the flat chrome format, whatever chrome format it actually is in. This means that for example, paths like chrome/browser/content/... match files under jar:chrome/browser.jar!/content/... in case of jar chrome format. ''' def __init__(self, *args, **kargs): FileFinder.__init__(self, *args, **kargs) self.files = FileRegistry() self.kind = 'flat' self.omnijar = None self.jarlogs = {} self.optimizedjars = False jars = set() for p, f in FileFinder.find(self, '*'): # Skip the precomplete file, which is generated at packaging time. if p == 'precomplete': continue base = mozpack.path.dirname(p) # If the file is a zip/jar that is not a .xpi, and contains a # chrome.manifest, it is an omnijar. All the files it contains # go in the directory containing the omnijar. Manifests are merged # if there is a corresponding manifest in the directory. if not p.endswith('.xpi') and self._maybe_zip(f) and \ (mozpack.path.basename(p) == self.omnijar or not self.omnijar): jar = self._open_jar(p, f) if 'chrome.manifest' in jar: self.kind = 'omni' self.omnijar = mozpack.path.basename(p) self._fill_with_omnijar(base, jar) continue # If the file is a manifest, scan its entries for some referencing # jar: urls. If there are some, the files contained in the jar they # point to, go under a directory named after the jar. if is_manifest(p): m = self.files[p] if self.files.contains(p) \ else ManifestFile(base) for e in parse_manifest(self.base, p, f.open()): m.add(self._handle_manifest_entry(e, jars)) if self.files.contains(p): continue f = m if not p in jars: self.files.add(p, f) def _fill_with_omnijar(self, base, jar): for j in jar: path = mozpack.path.join(base, j.filename) if is_manifest(j.filename): m = self.files[path] if self.files.contains(path) \ else ManifestFile(mozpack.path.dirname(path)) for e in parse_manifest(None, path, j): m.add(e) if not self.files.contains(path): self.files.add(path, m) continue else: self.files.add(path, DeflatedFile(j)) def _handle_manifest_entry(self, entry, jars): jarpath = None if isinstance(entry, ManifestEntryWithRelPath) and \ urlparse(entry.relpath).scheme == 'jar': jarpath, entry = self._unjarize(entry, entry.relpath) elif isinstance(entry, ManifestResource) and \ urlparse(entry.target).scheme == 'jar': jarpath, entry = self._unjarize(entry, entry.target) if jarpath: # Don't defer unpacking the jar file. If we already saw # it, take (and remove) it from the registry. If we # haven't, try to find it now. if self.files.contains(jarpath): jar = self.files[jarpath] self.files.remove(jarpath) else: jar = [f for p, f in FileFinder.find(self, jarpath)] assert len(jar) == 1 jar = jar[0] if not jarpath in jars: base = mozpack.path.splitext(jarpath)[0] for j in self._open_jar(jarpath, jar): self.files.add(mozpack.path.join(base, j.filename), DeflatedFile(j)) jars.add(jarpath) self.kind = 'jar' return entry def _open_jar(self, path, file): ''' Return a JarReader for the given BaseFile instance, keeping a log of the preloaded entries it has. ''' jar = JarReader(fileobj=file.open()) if jar.is_optimized: self.optimizedjars = True if jar.last_preloaded: jarlog = jar.entries.keys() self.jarlogs[path] = jarlog[:jarlog.index(jar.last_preloaded) + 1] return jar def find(self, path): for p in self.files.match(path): yield p, self.files[p] def _maybe_zip(self, file): ''' Return whether the given BaseFile looks like a ZIP/Jar. ''' header = file.open().read(8) return len(header) == 8 and (header[0:2] == 'PK' or header[4:6] == 'PK') def _unjarize(self, entry, relpath): ''' Transform a manifest entry pointing to chrome data in a jar in one pointing to the corresponding unpacked path. Return the jar path and the new entry. ''' base = entry.base jar, relpath = urlparse(relpath).path.split('!', 1) entry = entry.rebase(mozpack.path.join(base, 'jar:%s!' % jar)) \ .move(mozpack.path.join(base, mozpack.path.splitext(jar)[0])) \ .rebase(base) return mozpack.path.join(base, jar), entry
class TestFileRegistry(MatchTestTemplate, unittest.TestCase): def add(self, path): self.registry.add(path, GeneratedFile(path)) def do_check(self, pattern, result): self.checked = True if result: self.assertTrue(self.registry.contains(pattern)) else: self.assertFalse(self.registry.contains(pattern)) self.assertEqual(self.registry.match(pattern), result) def test_file_registry(self): self.registry = FileRegistry() self.registry.add('foo', GeneratedFile('foo')) bar = GeneratedFile('bar') self.registry.add('bar', bar) self.assertEqual(self.registry.paths(), ['foo', 'bar']) self.assertEqual(self.registry['bar'], bar) self.assertRaises(ErrorMessage, self.registry.add, 'foo', GeneratedFile('foo2')) self.assertRaises(ErrorMessage, self.registry.remove, 'qux') self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar', GeneratedFile('foobar')) self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar/baz', GeneratedFile('foobar')) self.assertEqual(self.registry.paths(), ['foo', 'bar']) self.registry.remove('foo') self.assertEqual(self.registry.paths(), ['bar']) self.registry.remove('bar') self.assertEqual(self.registry.paths(), []) self.prepare_match_test() self.do_match_test() self.assertTrue(self.checked) self.assertEqual(self.registry.paths(), [ 'bar', 'foo/bar', 'foo/baz', 'foo/qux/1', 'foo/qux/bar', 'foo/qux/2/test', 'foo/qux/2/test2', ]) self.registry.remove('foo/qux') self.assertEqual(self.registry.paths(), ['bar', 'foo/bar', 'foo/baz']) self.registry.add('foo/qux', GeneratedFile('fooqux')) self.assertEqual(self.registry.paths(), ['bar', 'foo/bar', 'foo/baz', 'foo/qux']) self.registry.remove('foo/b*') self.assertEqual(self.registry.paths(), ['bar', 'foo/qux']) self.assertEqual([f for f, c in self.registry], ['bar', 'foo/qux']) self.assertEqual(len(self.registry), 2) self.add('foo/.foo') self.assertTrue(self.registry.contains('foo/.foo'))
class TestFileRegistry(MatchTestTemplate, unittest.TestCase): def add(self, path): self.registry.add(path, GeneratedFile(path)) def do_check(self, pattern, result): self.checked = True if result: self.assertTrue(self.registry.contains(pattern)) else: self.assertFalse(self.registry.contains(pattern)) self.assertEqual(self.registry.match(pattern), result) def test_file_registry(self): self.registry = FileRegistry() self.registry.add('foo', GeneratedFile('foo')) bar = GeneratedFile('bar') self.registry.add('bar', bar) self.assertEqual(self.registry.paths(), ['foo', 'bar']) self.assertEqual(self.registry['bar'], bar) self.assertRaises(ErrorMessage, self.registry.add, 'foo', GeneratedFile('foo2')) self.assertRaises(ErrorMessage, self.registry.remove, 'qux') self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar', GeneratedFile('foobar')) self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar/baz', GeneratedFile('foobar')) self.assertEqual(self.registry.paths(), ['foo', 'bar']) self.registry.remove('foo') self.assertEqual(self.registry.paths(), ['bar']) self.registry.remove('bar') self.assertEqual(self.registry.paths(), []) self.prepare_match_test() self.do_match_test() self.assertTrue(self.checked) self.assertEqual(self.registry.paths(), [ 'bar', 'foo/bar', 'foo/baz', 'foo/qux/1', 'foo/qux/bar', 'foo/qux/2/test', 'foo/qux/2/test2', ]) self.registry.remove('foo/qux') self.assertEqual(self.registry.paths(), ['bar', 'foo/bar', 'foo/baz']) self.registry.add('foo/qux', GeneratedFile('fooqux')) self.assertEqual(self.registry.paths(), ['bar', 'foo/bar', 'foo/baz', 'foo/qux']) self.registry.remove('foo/b*') self.assertEqual(self.registry.paths(), ['bar', 'foo/qux']) self.assertEqual([f for f, c in self.registry], ['bar', 'foo/qux']) self.assertEqual(len(self.registry), 2) self.add('foo/.foo') self.assertTrue(self.registry.contains('foo/.foo')) def test_registry_paths(self): self.registry = FileRegistry() # Can't add a file if it requires a directory in place of a # file we also require. self.registry.add('foo', GeneratedFile('foo')) self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar', GeneratedFile('foobar')) # Can't add a file if we already have a directory there. self.registry.add('bar/baz', GeneratedFile('barbaz')) self.assertRaises(ErrorMessage, self.registry.add, 'bar', GeneratedFile('bar')) # Bump the count of things that require bar/ to 2. self.registry.add('bar/zot', GeneratedFile('barzot')) self.assertRaises(ErrorMessage, self.registry.add, 'bar', GeneratedFile('bar')) # Drop the count of things that require bar/ to 1. self.registry.remove('bar/baz') self.assertRaises(ErrorMessage, self.registry.add, 'bar', GeneratedFile('bar')) # Drop the count of things that require bar/ to 0. self.registry.remove('bar/zot') self.registry.add('bar/zot', GeneratedFile('barzot')) def test_required_directories(self): self.registry = FileRegistry() self.registry.add('foo', GeneratedFile('foo')) self.assertEqual(self.registry.required_directories(), set()) self.registry.add('bar/baz', GeneratedFile('barbaz')) self.assertEqual(self.registry.required_directories(), {'bar'}) self.registry.add('bar/zot', GeneratedFile('barzot')) self.assertEqual(self.registry.required_directories(), {'bar'}) self.registry.add('bar/zap/zot', GeneratedFile('barzapzot')) self.assertEqual(self.registry.required_directories(), {'bar', 'bar/zap'}) self.registry.remove('bar/zap/zot') self.assertEqual(self.registry.required_directories(), {'bar'}) self.registry.remove('bar/baz') self.assertEqual(self.registry.required_directories(), {'bar'}) self.registry.remove('bar/zot') self.assertEqual(self.registry.required_directories(), set()) self.registry.add('x/y/z', GeneratedFile('xyz')) self.assertEqual(self.registry.required_directories(), {'x', 'x/y'})
class UnpackFinder(BaseFinder): """ Special Finder object that treats the source package directory as if it were in the flat chrome format, whatever chrome format it actually is in. This means that for example, paths like chrome/browser/content/... match files under jar:chrome/browser.jar!/content/... in case of jar chrome format. The only argument to the constructor is a Finder instance or a path. The UnpackFinder is populated with files from this Finder instance, or with files from a FileFinder using the given path as its root. """ def __init__(self, source, omnijar_name=None, unpack_xpi=True): if isinstance(source, BaseFinder): self._finder = source else: self._finder = FileFinder(source) self.base = self._finder.base self.files = FileRegistry() self.kind = "flat" if omnijar_name: self.omnijar = omnijar_name else: # Can't include globally because of bootstrapping issues. from buildconfig import substs self.omnijar = substs.get("OMNIJAR_NAME", "omni.ja") self.jarlogs = {} self.compressed = False self._unpack_xpi = unpack_xpi jars = set() for p, f in self._finder.find("*"): # Skip the precomplete file, which is generated at packaging time. if p == "precomplete": continue base = mozpath.dirname(p) # If the file matches the omnijar pattern, it is an omnijar. # All the files it contains go in the directory containing the full # pattern. Manifests are merged if there is a corresponding manifest # in the directory. if self._maybe_zip(f) and mozpath.match(p, "**/%s" % self.omnijar): jar = self._open_jar(p, f) if "chrome.manifest" in jar: self.kind = "omni" self._fill_with_jar(p[: -len(self.omnijar) - 1], jar) continue # If the file is a manifest, scan its entries for some referencing # jar: urls. If there are some, the files contained in the jar they # point to, go under a directory named after the jar. if is_manifest(p): m = self.files[p] if self.files.contains(p) else ManifestFile(base) for e in parse_manifest( self.base, p, codecs.getreader("utf-8")(f.open()) ): m.add(self._handle_manifest_entry(e, jars)) if self.files.contains(p): continue f = m # If we're unpacking packed addons and the file is a packed addon, # unpack it under a directory named after the xpi. if self._unpack_xpi and p.endswith(".xpi") and self._maybe_zip(f): self._fill_with_jar(p[:-4], self._open_jar(p, f)) continue if p not in jars: self.files.add(p, f) def _fill_with_jar(self, base, jar): for j in jar: path = mozpath.join(base, j.filename) if is_manifest(j.filename): m = ( self.files[path] if self.files.contains(path) else ManifestFile(mozpath.dirname(path)) ) for e in parse_manifest(None, path, j): m.add(e) if not self.files.contains(path): self.files.add(path, m) continue else: self.files.add(path, DeflatedFile(j)) def _handle_manifest_entry(self, entry, jars): jarpath = None if ( isinstance(entry, ManifestEntryWithRelPath) and urlparse(entry.relpath).scheme == "jar" ): jarpath, entry = self._unjarize(entry, entry.relpath) elif ( isinstance(entry, ManifestResource) and urlparse(entry.target).scheme == "jar" ): jarpath, entry = self._unjarize(entry, entry.target) if jarpath: # Don't defer unpacking the jar file. If we already saw # it, take (and remove) it from the registry. If we # haven't, try to find it now. if self.files.contains(jarpath): jar = self.files[jarpath] self.files.remove(jarpath) else: jar = [f for p, f in self._finder.find(jarpath)] assert len(jar) == 1 jar = jar[0] if jarpath not in jars: base = mozpath.splitext(jarpath)[0] for j in self._open_jar(jarpath, jar): self.files.add(mozpath.join(base, j.filename), DeflatedFile(j)) jars.add(jarpath) self.kind = "jar" return entry def _open_jar(self, path, file): """ Return a JarReader for the given BaseFile instance, keeping a log of the preloaded entries it has. """ jar = JarReader(fileobj=file.open()) self.compressed = max(self.compressed, jar.compression) if jar.last_preloaded: jarlog = list(jar.entries.keys()) self.jarlogs[path] = jarlog[: jarlog.index(jar.last_preloaded) + 1] return jar def find(self, path): for p in self.files.match(path): yield p, self.files[p] def _maybe_zip(self, file): """ Return whether the given BaseFile looks like a ZIP/Jar. """ header = file.open().read(8) return len(header) == 8 and (header[0:2] == b"PK" or header[4:6] == b"PK") def _unjarize(self, entry, relpath): """ Transform a manifest entry pointing to chrome data in a jar in one pointing to the corresponding unpacked path. Return the jar path and the new entry. """ base = entry.base jar, relpath = urlparse(relpath).path.split("!", 1) entry = ( entry.rebase(mozpath.join(base, "jar:%s!" % jar)) .move(mozpath.join(base, mozpath.splitext(jar)[0])) .rebase(base) ) return mozpath.join(base, jar), entry