def __init__(self, copier, omnijar_name, compress=True, non_resources=()): PiecemealFormatter.__init__(self, copier) self._omnijar_name = omnijar_name self._compress = compress self._non_resources = non_resources self._sub_formatter[''] = FlatSubFormatter(copier) jarrer = Jarrer(self._compress) self._sub_formatter[omnijar_name] = FlatSubFormatter(jarrer)
def _add_base(self, base, addon=False): if addon is True: jarrer = Jarrer(self._compress) self.copier.add(base + ".xpi", jarrer) self._sub_formatter[base] = FlatSubFormatter(jarrer) else: self._sub_formatter[base] = JarSubFormatter( FileRegistrySubtree(base, self.copier), self._compress)
def add(self, path, content): chrome = self._chromepath(path) if chrome: jar = chrome + '.jar' if not self.copier.contains(jar): self.copier.add(jar, Jarrer(self._compress, self._optimize)) if not self.copier[jar].contains(mozpath.relpath(path, chrome)): self.copier[jar].add(mozpath.relpath(path, chrome), content) else: FlatFormatter.add(self, path, content)
def test_jarrer_compress(self): copier = Jarrer() copier.add("foo/bar", GeneratedFile(b"ffffff")) copier.add("foo/qux", GeneratedFile(b"ffffff"), compress=False) dest = MockDest() copier.copy(dest) self.check_jar(dest, copier) dest.seek(0) jar = JarReader(fileobj=dest) self.assertTrue(jar["foo/bar"].compressed) self.assertFalse(jar["foo/qux"].compressed)
def main(args): parser = argparse.ArgumentParser() parser.add_argument("-C", metavar='DIR', default=".", help="Change to given directory before considering " "other paths") parser.add_argument("--strip", action='store_true', help="Strip executables") parser.add_argument("-x", metavar='EXCLUDE', default=[], action='append', help="Exclude files that match the pattern") parser.add_argument("zip", help="Path to zip file to write") parser.add_argument("input", nargs="+", help="Path to files to add to zip") args = parser.parse_args(args) jarrer = Jarrer() with errors.accumulate(): finder = FileFinder(args.C, find_executables=args.strip) for path in args.input: for p, f in finder.find(path): if not any([match(p, exclude) for exclude in args.x]): jarrer.add(p, f) jarrer.copy(mozpath.join(args.C, args.zip))
def package_coverage_data(root, output_file): finder = FileFinder(root) jarrer = Jarrer() for p, f in finder.find("**/*.gcno"): jarrer.add(p, f) dist_include_manifest = mozpath.join(buildconfig.topobjdir, "_build_manifests", "install", "dist_include") linked_files = describe_install_manifest(dist_include_manifest, "dist/include") mapping_file = GeneratedFile(json.dumps(linked_files, sort_keys=True)) jarrer.add("linked-files-map.json", mapping_file) jarrer.copy(output_file)
def main(args): parser = argparse.ArgumentParser() parser.add_argument("-C", metavar='DIR', default=".", help="Change to given directory before considering " "other paths") parser.add_argument("--strip", action='store_true', help="Strip executables") parser.add_argument("-x", metavar='EXCLUDE', default=[], action='append', help="Exclude files that match the pattern") parser.add_argument("zip", help="Path to zip file to write") parser.add_argument("input", nargs="+", help="Path to files to add to zip") args = parser.parse_args(args) jarrer = Jarrer(optimize=False) with errors.accumulate(): finder = FileFinder(args.C, find_executables=args.strip) for path in args.input: for p, f in finder.find(path): if not any([match(p, exclude) for exclude in args.x]): jarrer.add(p, f) jarrer.copy(mozpath.join(args.C, args.zip))
def _get_omnijar(self, path, create=True): ''' Return the omnijar corresponding to the given path, its base directory and the path translated to be under the omnijar.. ''' base = self._get_base(path) if not base in self.omnijars: if not create: return None, '', path omnijar = Jarrer(self._compress, self._optimize) self.omnijars[base] = FlatFormatter(omnijar) self.copier.add(mozpack.path.join(base, self._omnijar_name), omnijar) return self.omnijars[base], base, mozpack.path.relpath(path, base)
def test_jarrer_compress(self): copier = Jarrer() copier.add('foo/bar', GeneratedFile('ffffff')) copier.add('foo/qux', GeneratedFile('ffffff'), compress=False) dest = MockDest() copier.copy(dest) self.check_jar(dest, copier) dest.seek(0) jar = JarReader(fileobj=dest) self.assertTrue(jar['foo/bar'].compressed) self.assertFalse(jar['foo/qux'].compressed)
def add_manifest(self, entry): if isinstance(entry, ManifestChrome) and \ not urlparse(entry.relpath).scheme: chromepath, entry = self._jarize(entry, entry.relpath) assert not self._frozen_chrome if chromepath not in self._sub_formatter: jarrer = Jarrer(self._compress, self._optimize) self.copier.add(chromepath + '.jar', jarrer) self._sub_formatter[chromepath] = FlatSubFormatter(jarrer) elif isinstance(entry, ManifestResource) and \ not urlparse(entry.target).scheme: chromepath, new_entry = self._jarize(entry, entry.target) if chromepath in self._sub_formatter: entry = new_entry PiecemealFormatter.add_manifest(self, entry)
def package_gcno_tree(root, output_file): # XXX JarWriter doesn't support unicode strings, see bug 1056859 if isinstance(root, unicode): root = root.encode('utf-8') finder = FileFinder(root) jarrer = Jarrer(optimize=False) for p, f in finder.find("**/*.gcno"): jarrer.add(p, f) jarrer.copy(output_file)
def package_coverage_data(root, output_file): # XXX JarWriter doesn't support unicode strings, see bug 1056859 if isinstance(root, unicode): root = root.encode('utf-8') finder = FileFinder(root) jarrer = Jarrer() for p, f in finder.find("**/*.gcno"): jarrer.add(p, f) dist_include_manifest = mozpath.join(buildconfig.topobjdir, '_build_manifests', 'install', 'dist_include') linked_files = describe_install_manifest(dist_include_manifest, 'dist/include') mapping_file = GeneratedFile(json.dumps(linked_files, sort_keys=True)) jarrer.add('linked-files-map.json', mapping_file) jarrer.copy(output_file)
def _get_formatter(self, path, is_resource=None): ''' Return the (sub)formatter corresponding to the given path, its base directory and the path relative to that base. ''' base = self._get_base(path) use_omnijar = base not in self._addons if use_omnijar: if is_resource is None: is_resource = self.is_resource(path, base) use_omnijar = is_resource if not use_omnijar: return super(OmniJarFormatter, self), '', path if not base in self.omnijars: omnijar = Jarrer(self._compress, self._optimize) self.omnijars[base] = FlatFormatter(omnijar) self.copier.add(mozpath.join(base, self._omnijar_name), omnijar) return self.omnijars[base], base, mozpath.relpath(path, base)
def build(self, dest): src = os.path.join(self.topsrcdir, 'services', 'sync', 'tps', 'extensions', 'tps') dest = os.path.join(dest or os.path.join(self.topobjdir, 'services', 'sync'), 'tps.xpi') if not os.path.exists(os.path.dirname(dest)): os.makedirs(os.path.dirname(dest)) jarrer = Jarrer(optimize=False) for p, f in FileFinder(src).find('*'): jarrer.add(p, f) jarrer.copy(dest) print('Built TPS add-on as %s' % dest)
def package_coverage_data(root, output_file): # XXX JarWriter doesn't support unicode strings, see bug 1056859 if isinstance(root, unicode): root = root.encode('utf-8') finder = FileFinder(root) jarrer = Jarrer(optimize=False) for p, f in finder.find("**/*.gcno"): jarrer.add(p, f) dist_include_manifest = mozpath.join(buildconfig.topobjdir, '_build_manifests', 'install', 'dist_include') linked_files = describe_install_manifest(dist_include_manifest, 'dist/include') mapping_file = GeneratedFile(json.dumps(linked_files, sort_keys=True)) jarrer.add('linked-files-map.json', mapping_file) jarrer.copy(output_file)
def main(args): parser = argparse.ArgumentParser() parser.add_argument("-C", metavar='DIR', default=".", help="Change to given directory before considering " "other paths") parser.add_argument("zip", help="Path to zip file to write") parser.add_argument("input", nargs="+", help="Path to files to add to zip") args = parser.parse_args(args) jarrer = Jarrer(optimize=False) with errors.accumulate(): finder = FileFinder(args.C) for path in args.input: for p, f in finder.find(path): jarrer.add(p, f) jarrer.copy(mozpath.join(args.C, args.zip))
def main(args): parser = argparse.ArgumentParser() parser.add_argument("--base-dir", default=os.path.join(buildconfig.topobjdir, "dist", "bin"), help="Store paths relative to this directory") parser.add_argument("zip", help="Path to zip file to write") parser.add_argument("input", nargs="+", help="Path to files to add to zip") args = parser.parse_args(args) jarrer = Jarrer(optimize=False) with errors.accumulate(): finder = FileFinder(args.base_dir) for i in args.input: path = mozpath.relpath(i, args.base_dir) for p, f in finder.find(path): jarrer.add(p, f) jarrer.copy(args.zip)
def build(self, dest): src = os.path.join( self.topsrcdir, "services", "sync", "tps", "extensions", "tps" ) dest = os.path.join( dest or os.path.join(self.topobjdir, "services", "sync"), "tps.xpi" ) if not os.path.exists(os.path.dirname(dest)): os.makedirs(os.path.dirname(dest)) if os.path.isfile(dest): os.unlink(dest) jarrer = Jarrer() for p, f in FileFinder(src).find("*"): jarrer.add(p, f) jarrer.copy(dest) print("Built TPS add-on as %s" % dest)
def package_fennec_apk(inputs=[], omni_ja=None, classes_dex=None, lib_dirs=[], assets_dirs=[], features_dirs=[], root_files=[], verbose=False): jarrer = Jarrer(optimize=False) # First, take input files. The contents of the later files overwrites the # content of earlier files. for input in inputs: jar = JarReader(input) for file in jar: path = file.filename if jarrer.contains(path): jarrer.remove(path) jarrer.add(path, DeflatedFile(file), compress=file.compressed) def add(path, file, compress=None): abspath = os.path.abspath(file.path) if verbose: print('Packaging %s from %s' % (path, file.path)) if not os.path.exists(abspath): raise ValueError('File %s not found (looked for %s)' % \ (file.path, abspath)) if jarrer.contains(path): jarrer.remove(path) jarrer.add(path, file, compress=compress) for features_dir in features_dirs: finder = FileFinder(features_dir, find_executables=False) for p, f in finder.find('**'): add(mozpath.join('assets', 'features', p), f, False) for assets_dir in assets_dirs: finder = FileFinder(assets_dir, find_executables=False) for p, f in finder.find('**'): compress = None # Take default from Jarrer. if p.endswith('.so'): # Asset libraries are special. if f.open().read(5)[1:] == '7zXZ': print('%s is already compressed' % p) # We need to store (rather than deflate) compressed libraries # (even if we don't compress them ourselves). compress = False elif buildconfig.substs.get('XZ'): cmd = [ buildconfig.substs.get('XZ'), '-zkf', mozpath.join(finder.base, p) ] bcj = None if buildconfig.substs.get('MOZ_THUMB2'): bcj = '--armthumb' elif buildconfig.substs.get('CPU_ARCH') == 'arm': bcj = '--arm' elif buildconfig.substs.get('CPU_ARCH') == 'x86': bcj = '--x86' if bcj: cmd.extend([bcj, '--lzma2']) print('xz-compressing %s with %s' % (p, ' '.join(cmd))) subprocess.check_output(cmd) os.rename(f.path + '.xz', f.path) compress = False add(mozpath.join('assets', p), f, compress=compress) for lib_dir in lib_dirs: finder = FileFinder(lib_dir, find_executables=False) for p, f in finder.find('**'): add(mozpath.join('lib', p), f) for root_file in root_files: add(os.path.basename(root_file), File(root_file)) if omni_ja: add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False) if classes_dex: add('classes.dex', File(classes_dex)) return jarrer
def package_fennec_apk(inputs=[], omni_ja=None, classes_dex=None, lib_dirs=[], assets_dirs=[], features_dirs=[], root_files=[], verbose=False): jarrer = Jarrer(optimize=False) # First, take input files. The contents of the later files overwrites the # content of earlier files. Multidexing requires special care: we want a # coherent set of classesN.dex files, so we only take DEX files from a # single input. This avoids taking, say, classes{1,2,3}.dex from the first # input and only classes{1,2}.dex from the second input, leading to # (potentially) duplicated symbols at runtime. last_input_with_dex_files = None for input in inputs: jar = JarReader(input) for file in jar: path = file.filename if mozpath.match(path, '/classes*.dex'): last_input_with_dex_files = input continue if jarrer.contains(path): jarrer.remove(path) jarrer.add(path, DeflatedFile(file), compress=file.compressed) # If we have an input with DEX files, take them all here. if last_input_with_dex_files: jar = JarReader(last_input_with_dex_files) for file in jar: path = file.filename if not mozpath.match(path, '/classes*.dex'): continue if jarrer.contains(path): jarrer.remove(path) jarrer.add(path, DeflatedFile(file), compress=file.compressed) def add(path, file, compress=None): abspath = os.path.abspath(file.path) if verbose: print('Packaging %s from %s' % (path, file.path)) if not os.path.exists(abspath): raise ValueError('File %s not found (looked for %s)' % \ (file.path, abspath)) if jarrer.contains(path): jarrer.remove(path) jarrer.add(path, file, compress=compress) for features_dir in features_dirs: finder = FileFinder(features_dir) for p, f in finder.find('**'): add(mozpath.join('assets', 'features', p), f, False) for assets_dir in assets_dirs: finder = FileFinder(assets_dir) for p, f in finder.find('**'): compress = None # Take default from Jarrer. if p.endswith('.so'): # Asset libraries are special. if f.open().read(5)[1:] == '7zXZ': print('%s is already compressed' % p) # We need to store (rather than deflate) compressed libraries # (even if we don't compress them ourselves). compress = False elif buildconfig.substs.get('XZ'): cmd = [buildconfig.substs.get('XZ'), '-zkf', mozpath.join(finder.base, p)] # For now, the mozglue XZStream ELF loader can only support xz files # with a single stream that contains a single block. In xz, there is no # explicit option to set the max block count. Instead, we force xz to use # single thread mode, which results in a single block. cmd.extend(['--threads=1']) bcj = None if buildconfig.substs.get('MOZ_THUMB2'): bcj = '--armthumb' elif buildconfig.substs.get('CPU_ARCH') == 'arm': bcj = '--arm' elif buildconfig.substs.get('CPU_ARCH') == 'x86': bcj = '--x86' if bcj: cmd.extend([bcj]) # We need to explicitly specify the LZMA filter chain to ensure consistent builds # across platforms. Note that the dict size must be less then 16MiB per the hardcoded # value in mozglue/linker/XZStream.cpp. This is the default LZMA filter chain for for # xz-utils version 5.0. See: # https://github.com/xz-mirror/xz/blob/v5.0.0/src/liblzma/lzma/lzma_encoder_presets.c # https://github.com/xz-mirror/xz/blob/v5.0.0/src/liblzma/api/lzma/container.h#L31 cmd.extend(['--lzma2=dict=8MiB,lc=3,lp=0,pb=2,mode=normal,nice=64,mf=bt4,depth=0']) print('xz-compressing %s with %s' % (p, ' '.join(cmd))) subprocess.check_output(cmd) os.rename(f.path + '.xz', f.path) compress = False add(mozpath.join('assets', p), f, compress=compress) for lib_dir in lib_dirs: finder = FileFinder(lib_dir) for p, f in finder.find('**'): add(mozpath.join('lib', p), f) for root_file in root_files: add(os.path.basename(root_file), File(root_file)) if omni_ja: add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False) if classes_dex: if buildconfig.substs.get('MOZ_BUILD_MOBILE_ANDROID_WITH_GRADLE'): raise ValueError("Fennec APKs built --with-gradle " "should never specify classes.dex") add('classes.dex', File(classes_dex)) return jarrer
def test_jarrer(self): copier = Jarrer() copier.add('foo/bar', GeneratedFile('foobar')) copier.add('foo/qux', GeneratedFile('fooqux')) copier.add('foo/deep/nested/directory/file', GeneratedFile('fooz')) copier.add('bar', GeneratedFile('bar')) copier.add('qux/foo', GeneratedFile('quxfoo')) copier.add('qux/bar', GeneratedFile('')) dest = MockDest() copier.copy(dest) self.check_jar(dest, copier) copier.remove('foo') copier.add('test', GeneratedFile('test')) copier.copy(dest) self.check_jar(dest, copier) copier.remove('test') copier.add('test', GeneratedFile('replaced-content')) copier.copy(dest) self.check_jar(dest, copier) copier.copy(dest) self.check_jar(dest, copier) preloaded = ['qux/bar', 'bar'] copier.preload(preloaded) copier.copy(dest) dest.seek(0) jar = JarReader(fileobj=dest) self.assertEqual([f.filename for f in jar], preloaded + [p for p in copier.paths() if not p in preloaded]) self.assertEqual(jar.last_preloaded, preloaded[-1])
def test_jarrer(self): copier = Jarrer() copier.add("foo/bar", GeneratedFile(b"foobar")) copier.add("foo/qux", GeneratedFile(b"fooqux")) copier.add("foo/deep/nested/directory/file", GeneratedFile(b"fooz")) copier.add("bar", GeneratedFile(b"bar")) copier.add("qux/foo", GeneratedFile(b"quxfoo")) copier.add("qux/bar", GeneratedFile(b"")) dest = MockDest() copier.copy(dest) self.check_jar(dest, copier) copier.remove("foo") copier.add("test", GeneratedFile(b"test")) copier.copy(dest) self.check_jar(dest, copier) copier.remove("test") copier.add("test", GeneratedFile(b"replaced-content")) copier.copy(dest) self.check_jar(dest, copier) copier.copy(dest) self.check_jar(dest, copier) preloaded = ["qux/bar", "bar"] copier.preload(preloaded) copier.copy(dest) dest.seek(0) jar = JarReader(fileobj=dest) self.assertEqual( [f.filename for f in jar], preloaded + [p for p in copier.paths() if p not in preloaded], ) self.assertEqual(jar.last_preloaded, preloaded[-1])
def package_geckoview_aar(topsrcdir, distdir, appname, output_file): jarrer = Jarrer(optimize=False) app_path = os.path.join(distdir, appname) assets = FileFinder(os.path.join(app_path, "assets"), ignore=["*.so"]) for p, f in assets.find("omni.ja"): jarrer.add(os.path.join("assets", p), f) # The folder that contains Fennec's JAR files and resources. base_path = os.path.join(distdir, "..", "mobile", "android", "base") # The resource set is packaged during Fennec's build. resjar = JarReader(os.path.join(base_path, "geckoview_resources.zip")) for p, f in JarFinder(base_path, resjar).find("*"): jarrer.add(os.path.join("res", p), f) # Package the contents of all Fennec JAR files into classes.jar. classes_jar_file = _generate_geckoview_classes_jar(distdir, base_path) jarrer.add("classes.jar", classes_jar_file) # Add R.txt. jarrer.add("R.txt", File(os.path.join(base_path, "R.txt"))) # Finally add AndroidManifest.xml. srcdir = os.path.join(topsrcdir, "mobile", "android", "geckoview_library", "geckoview") jarrer.add("AndroidManifest.xml", File(os.path.join(srcdir, "AndroidManifest.xml"))) jarrer.copy(output_file) return 0
def package_geckolibs_aar(topsrcdir, distdir, appname, output_file): jarrer = Jarrer(optimize=False) srcdir = os.path.join(topsrcdir, 'mobile', 'android', 'geckoview_library', 'geckolibs') jarrer.add('AndroidManifest.xml', File(os.path.join(srcdir, 'AndroidManifest.xml'))) jarrer.add('classes.jar', File(os.path.join(srcdir, 'classes.jar'))) jni = FileFinder(os.path.join(distdir, appname, 'lib')) for p, f in jni.find('**/*.so'): jarrer.add(os.path.join('jni', p), f) # Include the buildinfo JSON as an asset, to give future consumers at least # a hope of determining where this AAR came from. json = FileFinder(distdir, ignore=['*.mozinfo.json']) for p, f in json.find('*.json'): jarrer.add(os.path.join('assets', p), f) # This neatly ignores omni.ja. assets = FileFinder(os.path.join(distdir, appname, 'assets')) for p, f in assets.find('**/*.so'): jarrer.add(os.path.join('assets', p), f) jarrer.copy(output_file) return 0
def package_geckolibs_aar(topsrcdir, distdir, appname, output_file): jarrer = Jarrer(optimize=False) srcdir = os.path.join(topsrcdir, "mobile", "android", "geckoview_library", "geckolibs") jarrer.add("AndroidManifest.xml", File(os.path.join(srcdir, "AndroidManifest.xml"))) jarrer.add("classes.jar", File(os.path.join(srcdir, "classes.jar"))) jni = FileFinder(os.path.join(distdir, appname, "lib")) for p, f in jni.find("**/*.so"): jarrer.add(os.path.join("jni", p), f) # Include the buildinfo JSON as an asset, to give future consumers at least # a hope of determining where this AAR came from. json = FileFinder(distdir, ignore=["*.mozinfo.json"]) for p, f in json.find("*.json"): jarrer.add(os.path.join("assets", p), f) # This neatly ignores omni.ja. assets = FileFinder(os.path.join(distdir, appname, "assets")) for p, f in assets.find("**/*.so"): jarrer.add(os.path.join("assets", p), f) jarrer.copy(output_file) return 0
def package_fennec_apk(inputs=[], omni_ja=None, classes_dex=None, lib_dirs=[], assets_dirs=[], features_dirs=[], root_files=[], verbose=False): jarrer = Jarrer(optimize=False) # First, take input files. The contents of the later files overwrites the # content of earlier files. Multidexing requires special care: we want a # coherent set of classesN.dex files, so we only take DEX files from a # single input. This avoids taking, say, classes{1,2,3}.dex from the first # input and only classes{1,2}.dex from the second input, leading to # (potentially) duplicated symbols at runtime. last_input_with_dex_files = None for input in inputs: jar = JarReader(input) for file in jar: path = file.filename if mozpath.match(path, '/classes*.dex'): last_input_with_dex_files = input continue if jarrer.contains(path): jarrer.remove(path) jarrer.add(path, DeflatedFile(file), compress=file.compressed) # If we have an input with DEX files, take them all here. if last_input_with_dex_files: jar = JarReader(last_input_with_dex_files) for file in jar: path = file.filename if not mozpath.match(path, '/classes*.dex'): continue if jarrer.contains(path): jarrer.remove(path) jarrer.add(path, DeflatedFile(file), compress=file.compressed) def add(path, file, compress=None): abspath = os.path.abspath(file.path) if verbose: print('Packaging %s from %s' % (path, file.path)) if not os.path.exists(abspath): raise ValueError('File %s not found (looked for %s)' % \ (file.path, abspath)) if jarrer.contains(path): jarrer.remove(path) jarrer.add(path, file, compress=compress) for features_dir in features_dirs: finder = FileFinder(features_dir) for p, f in finder.find('**'): add(mozpath.join('assets', 'features', p), f, False) for assets_dir in assets_dirs: finder = FileFinder(assets_dir) for p, f in finder.find('**'): compress = None # Take default from Jarrer. if p.endswith('.so'): # Asset libraries are special. if f.open().read(5)[1:] == '7zXZ': print('%s is already compressed' % p) # We need to store (rather than deflate) compressed libraries # (even if we don't compress them ourselves). compress = False elif buildconfig.substs.get('XZ'): cmd = [ buildconfig.substs.get('XZ'), '-zkf', mozpath.join(finder.base, p) ] # For now, the mozglue XZStream ELF loader can only support xz files # with a single stream that contains a single block. In xz, there is no # explicit option to set the max block count. Instead, we force xz to use # single thread mode, which results in a single block. cmd.extend(['--threads=1']) bcj = None if buildconfig.substs.get('MOZ_THUMB2'): bcj = '--armthumb' elif buildconfig.substs.get('CPU_ARCH') == 'arm': bcj = '--arm' elif buildconfig.substs.get('CPU_ARCH') == 'x86': bcj = '--x86' if bcj: cmd.extend([bcj]) # We need to explicitly specify the LZMA filter chain to ensure consistent builds # across platforms. Note that the dict size must be less then 16MiB per the hardcoded # value in mozglue/linker/XZStream.cpp. This is the default LZMA filter chain for for # xz-utils version 5.0. See: # https://github.com/xz-mirror/xz/blob/v5.0.0/src/liblzma/lzma/lzma_encoder_presets.c # https://github.com/xz-mirror/xz/blob/v5.0.0/src/liblzma/api/lzma/container.h#L31 cmd.extend([ '--lzma2=dict=8MiB,lc=3,lp=0,pb=2,mode=normal,nice=64,mf=bt4,depth=0' ]) print('xz-compressing %s with %s' % (p, ' '.join(cmd))) subprocess.check_output(cmd) os.rename(f.path + '.xz', f.path) compress = False add(mozpath.join('assets', p), f, compress=compress) for lib_dir in lib_dirs: finder = FileFinder(lib_dir) for p, f in finder.find('**'): add(mozpath.join('lib', p), f) for root_file in root_files: add(os.path.basename(root_file), File(root_file)) if omni_ja: add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False) if classes_dex: if buildconfig.substs.get('MOZ_BUILD_MOBILE_ANDROID_WITH_GRADLE'): raise ValueError("Fennec APKs built --with-gradle " "should never specify classes.dex") add('classes.dex', File(classes_dex)) return jarrer
def package_geckoview_aar(topsrcdir, distdir, appname, output_file): jarrer = Jarrer(optimize=False) app_path = os.path.join(distdir, appname) assets = FileFinder(os.path.join(app_path, 'assets'), ignore=['*.so']) for p, f in assets.find('omni.ja'): jarrer.add(os.path.join('assets', p), f) # The folder that contains Fennec's JAR files and resources. base_path = os.path.join(distdir, '..', 'mobile', 'android', 'base') # The resource set is packaged during Fennec's build. resjar = JarReader(os.path.join(base_path, 'geckoview_resources.zip')) for p, f in JarFinder(base_path, resjar).find('*'): jarrer.add(os.path.join('res', p), f) # Package the contents of all Fennec JAR files into classes.jar. classes_jar_file = _generate_geckoview_classes_jar(distdir, base_path) jarrer.add('classes.jar', classes_jar_file) # Add R.txt. jarrer.add('R.txt', File(os.path.join(base_path, 'R.txt'))) # Finally add AndroidManifest.xml. srcdir = os.path.join(topsrcdir, 'mobile', 'android', 'geckoview_library', 'geckoview') jarrer.add('AndroidManifest.xml', File(os.path.join(srcdir, 'AndroidManifest.xml'))) jarrer.copy(output_file) return 0
def package_fennec_apk(inputs=[], omni_ja=None, classes_dex=None, lib_dirs=[], assets_dirs=[], szip_assets_libs_with=None, root_files=[], verbose=False): jarrer = Jarrer(optimize=False) # First, take input files. The contents of the later files overwrites the # content of earlier files. for input in inputs: jar = JarReader(input) for file in jar: path = file.filename if jarrer.contains(path): jarrer.remove(path) jarrer.add(path, DeflatedFile(file), compress=file.compressed) def add(path, file, compress=None): abspath = os.path.abspath(file.path) if verbose: print('Packaging %s from %s' % (path, file.path)) if not os.path.exists(abspath): raise ValueError('File %s not found (looked for %s)' % \ (file.path, abspath)) if jarrer.contains(path): jarrer.remove(path) jarrer.add(path, file, compress=compress) for assets_dir in assets_dirs: finder = FileFinder(assets_dir, find_executables=False) for p, f in finder.find('**'): compress = None # Take default from Jarrer. if p.endswith('.so'): # Asset libraries are special. if szip_assets_libs_with: # We need to szip libraries before packing. The file # returned by the finder is not yet opened. When it is # opened, it will "see" the content updated by szip. subprocess.check_output([szip_assets_libs_with, mozpath.join(finder.base, p)]) if f.open().read(4) == 'SeZz': # We need to store (rather than deflate) szipped libraries # (even if we don't szip them ourselves). compress = False add(mozpath.join('assets', p), f, compress=compress) for lib_dir in lib_dirs: finder = FileFinder(lib_dir, find_executables=False) for p, f in finder.find('**'): add(mozpath.join('lib', p), f) for root_file in root_files: add(os.path.basename(root_file), File(root_file)) if omni_ja: add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False) if classes_dex: add('classes.dex', File(classes_dex)) return jarrer
def package_geckolibs_aar(topsrcdir, distdir, output_file): jarrer = Jarrer(optimize=False) srcdir = os.path.join(topsrcdir, 'mobile', 'android', 'geckoview_library', 'geckolibs') jarrer.add('AndroidManifest.xml', File(os.path.join(srcdir, 'AndroidManifest.xml'))) jarrer.add('classes.jar', File(os.path.join(srcdir, 'classes.jar'))) jni = FileFinder(os.path.join(distdir, 'fennec', 'lib')) for p, f in jni.find('**/*.so'): jarrer.add(os.path.join('jni', p), f) # Include the buildinfo JSON as an asset, to give future consumers at least # a hope of determining where this AAR came from. json = FileFinder(distdir, ignore=['*.mozinfo.json']) for p, f in json.find('*.json'): jarrer.add(os.path.join('assets', p), f) # This neatly ignores omni.ja. assets = FileFinder(os.path.join(distdir, 'fennec', 'assets')) for p, f in assets.find('**/*.so'): jarrer.add(os.path.join('assets', p), f) jarrer.copy(output_file) return 0
def package_geckoview_aar(topsrcdir, distdir, output_file): jarrer = Jarrer(optimize=False) fennec_path = os.path.join(distdir, 'fennec') assets = FileFinder(os.path.join(fennec_path, 'assets'), ignore=['*.so']) for p, f in assets.find('omni.ja'): jarrer.add(os.path.join('assets', p), f) # The folder that contains Fennec's JAR files and resources. base_path = os.path.join(distdir, '..', 'mobile', 'android', 'base') # The resource set is packaged during Fennec's build. resjar = JarReader(os.path.join(base_path, 'geckoview_resources.zip')) for p, f in JarFinder(p, resjar).find('*'): jarrer.add(os.path.join('res', p), f) # Package the contents of all Fennec JAR files into classes.jar. classes_jar_file = _generate_geckoview_classes_jar(distdir, base_path) jarrer.add('classes.jar', classes_jar_file) # Add R.txt. jarrer.add('R.txt', File(os.path.join(base_path, 'R.txt'))) # Finally add AndroidManifest.xml. srcdir = os.path.join(topsrcdir, 'mobile', 'android', 'geckoview_library', 'geckoview') jarrer.add('AndroidManifest.xml', File(os.path.join(srcdir, 'AndroidManifest.xml'))) jarrer.copy(output_file) return 0
def package_fennec_apk(inputs=[], omni_ja=None, lib_dirs=[], assets_dirs=[], features_dirs=[], root_files=[], verbose=False): jarrer = Jarrer() # First, take input files. The contents of the later files overwrites the # content of earlier files. Multidexing requires special care: we want a # coherent set of classesN.dex files, so we only take DEX files from a # single input. This avoids taking, say, classes{1,2,3}.dex from the first # input and only classes{1,2}.dex from the second input, leading to # (potentially) duplicated symbols at runtime. last_input_with_dex_files = None for input in inputs: jar = JarReader(input) for file in jar: path = file.filename if mozpath.match(path, '/classes*.dex'): last_input_with_dex_files = input continue if jarrer.contains(path): jarrer.remove(path) jarrer.add(path, DeflatedFile(file), compress=file.compressed) # If we have an input with DEX files, take them all here. if last_input_with_dex_files: jar = JarReader(last_input_with_dex_files) for file in jar: path = file.filename if not mozpath.match(path, '/classes*.dex'): continue if jarrer.contains(path): jarrer.remove(path) jarrer.add(path, DeflatedFile(file), compress=file.compressed) def add(path, file, compress=None): abspath = os.path.abspath(file.path) if verbose: print('Packaging %s from %s' % (path, file.path)) if not os.path.exists(abspath): raise ValueError('File %s not found (looked for %s)' % (file.path, abspath)) if jarrer.contains(path): jarrer.remove(path) jarrer.add(path, file, compress=compress) for features_dir in features_dirs: finder = FileFinder(features_dir) for p, f in finder.find('**'): add(mozpath.join('assets', 'features', p), f, False) for assets_dir in assets_dirs: finder = FileFinder(assets_dir) for p, f in finder.find('**'): add(mozpath.join('assets', p), f) for lib_dir in lib_dirs: finder = FileFinder(lib_dir) for p, f in finder.find('**'): add(mozpath.join('lib', p), f) for root_file in root_files: add(os.path.basename(root_file), File(root_file)) if omni_ja: add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False) return jarrer
def test_jarrer(self): copier = Jarrer() copier.add("foo/bar", GeneratedFile("foobar")) copier.add("foo/qux", GeneratedFile("fooqux")) copier.add("foo/deep/nested/directory/file", GeneratedFile("fooz")) copier.add("bar", GeneratedFile("bar")) copier.add("qux/foo", GeneratedFile("quxfoo")) copier.add("qux/bar", GeneratedFile("")) dest = MockDest() copier.copy(dest) self.check_jar(dest, copier) copier.remove("foo") copier.add("test", GeneratedFile("test")) copier.copy(dest) self.check_jar(dest, copier) copier.remove("test") copier.add("test", GeneratedFile("replaced-content")) copier.copy(dest) self.check_jar(dest, copier) copier.copy(dest) self.check_jar(dest, copier) preloaded = ["qux/bar", "bar"] copier.preload(preloaded) copier.copy(dest) dest.seek(0) jar = JarReader(fileobj=dest) self.assertEqual([f.filename for f in jar], preloaded + [p for p in copier.paths() if not p in preloaded]) self.assertEqual(jar.last_preloaded, preloaded[-1])
def package_fennec_apk(inputs=[], omni_ja=None, classes_dex=None, lib_dirs=[], assets_dirs=[], szip_assets_libs_with=None, root_files=[], verbose=False): jarrer = Jarrer(optimize=False) # First, take input files. The contents of the later files overwrites the # content of earlier files. for input in inputs: jar = JarReader(input) for file in jar: path = file.filename if jarrer.contains(path): jarrer.remove(path) jarrer.add(path, DeflatedFile(file), compress=file.compressed) def add(path, file, compress=None): abspath = os.path.abspath(file.path) if verbose: print('Packaging %s from %s' % (path, file.path)) if not os.path.exists(abspath): raise ValueError('File %s not found (looked for %s)' % \ (file.path, abspath)) if jarrer.contains(path): jarrer.remove(path) jarrer.add(path, file, compress=compress) for assets_dir in assets_dirs: finder = FileFinder(assets_dir, find_executables=False) for p, f in finder.find('**'): compress = None # Take default from Jarrer. if p.endswith('.so'): # Asset libraries are special. if szip_assets_libs_with: # We need to szip libraries before packing. The file # returned by the finder is not yet opened. When it is # opened, it will "see" the content updated by szip. subprocess.check_output( [szip_assets_libs_with, mozpath.join(finder.base, p)]) if f.open().read(4) == 'SeZz': # We need to store (rather than deflate) szipped libraries # (even if we don't szip them ourselves). compress = False add(mozpath.join('assets', p), f, compress=compress) for lib_dir in lib_dirs: finder = FileFinder(lib_dir, find_executables=False) for p, f in finder.find('**'): add(mozpath.join('lib', p), f) for root_file in root_files: add(os.path.basename(root_file), File(root_file)) if omni_ja: add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False) if classes_dex: add('classes.dex', File(classes_dex)) return jarrer
def package_fennec_apk(inputs=[], omni_ja=None, classes_dex=None, lib_dirs=[], assets_dirs=[], features_dirs=[], root_files=[], verbose=False): jarrer = Jarrer(optimize=False) # First, take input files. The contents of the later files overwrites the # content of earlier files. for input in inputs: jar = JarReader(input) for file in jar: path = file.filename if jarrer.contains(path): jarrer.remove(path) jarrer.add(path, DeflatedFile(file), compress=file.compressed) def add(path, file, compress=None): abspath = os.path.abspath(file.path) if verbose: print('Packaging %s from %s' % (path, file.path)) if not os.path.exists(abspath): raise ValueError('File %s not found (looked for %s)' % \ (file.path, abspath)) if jarrer.contains(path): jarrer.remove(path) jarrer.add(path, file, compress=compress) for features_dir in features_dirs: finder = FileFinder(features_dir) for p, f in finder.find('**'): add(mozpath.join('assets', 'features', p), f, False) for assets_dir in assets_dirs: finder = FileFinder(assets_dir) for p, f in finder.find('**'): compress = None # Take default from Jarrer. if p.endswith('.so'): # Asset libraries are special. if f.open().read(5)[1:] == '7zXZ': print('%s is already compressed' % p) # We need to store (rather than deflate) compressed libraries # (even if we don't compress them ourselves). compress = False elif buildconfig.substs.get('XZ'): cmd = [buildconfig.substs.get('XZ'), '-zkf', mozpath.join(finder.base, p)] bcj = None if buildconfig.substs.get('MOZ_THUMB2'): bcj = '--armthumb' elif buildconfig.substs.get('CPU_ARCH') == 'arm': bcj = '--arm' elif buildconfig.substs.get('CPU_ARCH') == 'x86': bcj = '--x86' if bcj: cmd.extend([bcj, '--lzma2']) print('xz-compressing %s with %s' % (p, ' '.join(cmd))) subprocess.check_output(cmd) os.rename(f.path + '.xz', f.path) compress = False add(mozpath.join('assets', p), f, compress=compress) for lib_dir in lib_dirs: finder = FileFinder(lib_dir) for p, f in finder.find('**'): add(mozpath.join('lib', p), f) for root_file in root_files: add(os.path.basename(root_file), File(root_file)) if omni_ja: add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False) if classes_dex: add('classes.dex', File(classes_dex)) return jarrer