def __init__(self, copier, omnijar_name, compress=True, non_resources=()):
PiecemealFormatter.__init__(self, copier)
self._omnijar_name = omnijar_name
self._compress = compress
self._non_resources = non_resources
self._sub_formatter[''] = FlatSubFormatter(copier)
jarrer = Jarrer(self._compress)
self._sub_formatter[omnijar_name] = FlatSubFormatter(jarrer)
def _add_base(self, base, addon=False):
if addon is True:
jarrer = Jarrer(self._compress)
self.copier.add(base + ".xpi", jarrer)
self._sub_formatter[base] = FlatSubFormatter(jarrer)
else:
self._sub_formatter[base] = JarSubFormatter(
FileRegistrySubtree(base, self.copier), self._compress)
def add(self, path, content):
chrome = self._chromepath(path)
if chrome:
jar = chrome + '.jar'
if not self.copier.contains(jar):
self.copier.add(jar, Jarrer(self._compress, self._optimize))
if not self.copier[jar].contains(mozpath.relpath(path, chrome)):
self.copier[jar].add(mozpath.relpath(path, chrome), content)
else:
FlatFormatter.add(self, path, content)
def test_jarrer_compress(self):
copier = Jarrer()
copier.add("foo/bar", GeneratedFile(b"ffffff"))
copier.add("foo/qux", GeneratedFile(b"ffffff"), compress=False)
dest = MockDest()
copier.copy(dest)
self.check_jar(dest, copier)
dest.seek(0)
jar = JarReader(fileobj=dest)
self.assertTrue(jar["foo/bar"].compressed)
self.assertFalse(jar["foo/qux"].compressed)
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument("-C",
metavar='DIR',
default=".",
help="Change to given directory before considering "
"other paths")
parser.add_argument("--strip",
action='store_true',
help="Strip executables")
parser.add_argument("-x",
metavar='EXCLUDE',
default=[],
action='append',
help="Exclude files that match the pattern")
parser.add_argument("zip", help="Path to zip file to write")
parser.add_argument("input", nargs="+", help="Path to files to add to zip")
args = parser.parse_args(args)
jarrer = Jarrer()
with errors.accumulate():
finder = FileFinder(args.C, find_executables=args.strip)
for path in args.input:
for p, f in finder.find(path):
if not any([match(p, exclude) for exclude in args.x]):
jarrer.add(p, f)
jarrer.copy(mozpath.join(args.C, args.zip))
def package_coverage_data(root, output_file):
finder = FileFinder(root)
jarrer = Jarrer()
for p, f in finder.find("**/*.gcno"):
jarrer.add(p, f)
dist_include_manifest = mozpath.join(buildconfig.topobjdir,
"_build_manifests", "install",
"dist_include")
linked_files = describe_install_manifest(dist_include_manifest,
"dist/include")
mapping_file = GeneratedFile(json.dumps(linked_files, sort_keys=True))
jarrer.add("linked-files-map.json", mapping_file)
jarrer.copy(output_file)
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument("-C", metavar='DIR', default=".",
help="Change to given directory before considering "
"other paths")
parser.add_argument("--strip", action='store_true',
help="Strip executables")
parser.add_argument("-x", metavar='EXCLUDE', default=[], action='append',
help="Exclude files that match the pattern")
parser.add_argument("zip", help="Path to zip file to write")
parser.add_argument("input", nargs="+",
help="Path to files to add to zip")
args = parser.parse_args(args)
jarrer = Jarrer(optimize=False)
with errors.accumulate():
finder = FileFinder(args.C, find_executables=args.strip)
for path in args.input:
for p, f in finder.find(path):
if not any([match(p, exclude) for exclude in args.x]):
jarrer.add(p, f)
jarrer.copy(mozpath.join(args.C, args.zip))
def _get_omnijar(self, path, create=True):
'''
Return the omnijar corresponding to the given path, its base directory
and the path translated to be under the omnijar..
'''
base = self._get_base(path)
if not base in self.omnijars:
if not create:
return None, '', path
omnijar = Jarrer(self._compress, self._optimize)
self.omnijars[base] = FlatFormatter(omnijar)
self.copier.add(mozpack.path.join(base, self._omnijar_name),
omnijar)
return self.omnijars[base], base, mozpack.path.relpath(path, base)
def test_jarrer_compress(self):
copier = Jarrer()
copier.add('foo/bar', GeneratedFile('ffffff'))
copier.add('foo/qux', GeneratedFile('ffffff'), compress=False)
dest = MockDest()
copier.copy(dest)
self.check_jar(dest, copier)
dest.seek(0)
jar = JarReader(fileobj=dest)
self.assertTrue(jar['foo/bar'].compressed)
self.assertFalse(jar['foo/qux'].compressed)
def add_manifest(self, entry):
if isinstance(entry, ManifestChrome) and \
not urlparse(entry.relpath).scheme:
chromepath, entry = self._jarize(entry, entry.relpath)
assert not self._frozen_chrome
if chromepath not in self._sub_formatter:
jarrer = Jarrer(self._compress, self._optimize)
self.copier.add(chromepath + '.jar', jarrer)
self._sub_formatter[chromepath] = FlatSubFormatter(jarrer)
elif isinstance(entry, ManifestResource) and \
not urlparse(entry.target).scheme:
chromepath, new_entry = self._jarize(entry, entry.target)
if chromepath in self._sub_formatter:
entry = new_entry
PiecemealFormatter.add_manifest(self, entry)
def package_gcno_tree(root, output_file):
# XXX JarWriter doesn't support unicode strings, see bug 1056859
if isinstance(root, unicode):
root = root.encode('utf-8')
finder = FileFinder(root)
jarrer = Jarrer(optimize=False)
for p, f in finder.find("**/*.gcno"):
jarrer.add(p, f)
jarrer.copy(output_file)
def package_coverage_data(root, output_file):
# XXX JarWriter doesn't support unicode strings, see bug 1056859
if isinstance(root, unicode):
root = root.encode('utf-8')
finder = FileFinder(root)
jarrer = Jarrer()
for p, f in finder.find("**/*.gcno"):
jarrer.add(p, f)
dist_include_manifest = mozpath.join(buildconfig.topobjdir,
'_build_manifests', 'install',
'dist_include')
linked_files = describe_install_manifest(dist_include_manifest,
'dist/include')
mapping_file = GeneratedFile(json.dumps(linked_files, sort_keys=True))
jarrer.add('linked-files-map.json', mapping_file)
jarrer.copy(output_file)
def _get_formatter(self, path, is_resource=None):
'''
Return the (sub)formatter corresponding to the given path, its base
directory and the path relative to that base.
'''
base = self._get_base(path)
use_omnijar = base not in self._addons
if use_omnijar:
if is_resource is None:
is_resource = self.is_resource(path, base)
use_omnijar = is_resource
if not use_omnijar:
return super(OmniJarFormatter, self), '', path
if not base in self.omnijars:
omnijar = Jarrer(self._compress, self._optimize)
self.omnijars[base] = FlatFormatter(omnijar)
self.copier.add(mozpath.join(base, self._omnijar_name), omnijar)
return self.omnijars[base], base, mozpath.relpath(path, base)
def build(self, dest):
src = os.path.join(self.topsrcdir, 'services', 'sync', 'tps', 'extensions', 'tps')
dest = os.path.join(dest or os.path.join(self.topobjdir, 'services', 'sync'), 'tps.xpi')
if not os.path.exists(os.path.dirname(dest)):
os.makedirs(os.path.dirname(dest))
jarrer = Jarrer(optimize=False)
for p, f in FileFinder(src).find('*'):
jarrer.add(p, f)
jarrer.copy(dest)
print('Built TPS add-on as %s' % dest)
def package_coverage_data(root, output_file):
# XXX JarWriter doesn't support unicode strings, see bug 1056859
if isinstance(root, unicode):
root = root.encode('utf-8')
finder = FileFinder(root)
jarrer = Jarrer(optimize=False)
for p, f in finder.find("**/*.gcno"):
jarrer.add(p, f)
dist_include_manifest = mozpath.join(buildconfig.topobjdir,
'_build_manifests',
'install',
'dist_include')
linked_files = describe_install_manifest(dist_include_manifest,
'dist/include')
mapping_file = GeneratedFile(json.dumps(linked_files, sort_keys=True))
jarrer.add('linked-files-map.json', mapping_file)
jarrer.copy(output_file)
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument("-C", metavar='DIR', default=".",
help="Change to given directory before considering "
"other paths")
parser.add_argument("zip", help="Path to zip file to write")
parser.add_argument("input", nargs="+",
help="Path to files to add to zip")
args = parser.parse_args(args)
jarrer = Jarrer(optimize=False)
with errors.accumulate():
finder = FileFinder(args.C)
for path in args.input:
for p, f in finder.find(path):
jarrer.add(p, f)
jarrer.copy(mozpath.join(args.C, args.zip))
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument("--base-dir",
default=os.path.join(buildconfig.topobjdir, "dist",
"bin"),
help="Store paths relative to this directory")
parser.add_argument("zip", help="Path to zip file to write")
parser.add_argument("input", nargs="+", help="Path to files to add to zip")
args = parser.parse_args(args)
jarrer = Jarrer(optimize=False)
with errors.accumulate():
finder = FileFinder(args.base_dir)
for i in args.input:
path = mozpath.relpath(i, args.base_dir)
for p, f in finder.find(path):
jarrer.add(p, f)
jarrer.copy(args.zip)
def build(self, dest):
src = os.path.join(
self.topsrcdir, "services", "sync", "tps", "extensions", "tps"
)
dest = os.path.join(
dest or os.path.join(self.topobjdir, "services", "sync"), "tps.xpi"
)
if not os.path.exists(os.path.dirname(dest)):
os.makedirs(os.path.dirname(dest))
if os.path.isfile(dest):
os.unlink(dest)
jarrer = Jarrer()
for p, f in FileFinder(src).find("*"):
jarrer.add(p, f)
jarrer.copy(dest)
print("Built TPS add-on as %s" % dest)
def package_fennec_apk(inputs=[],
omni_ja=None,
classes_dex=None,
lib_dirs=[],
assets_dirs=[],
features_dirs=[],
root_files=[],
verbose=False):
jarrer = Jarrer(optimize=False)
# First, take input files. The contents of the later files overwrites the
# content of earlier files.
for input in inputs:
jar = JarReader(input)
for file in jar:
path = file.filename
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, DeflatedFile(file), compress=file.compressed)
def add(path, file, compress=None):
abspath = os.path.abspath(file.path)
if verbose:
print('Packaging %s from %s' % (path, file.path))
if not os.path.exists(abspath):
raise ValueError('File %s not found (looked for %s)' % \
(file.path, abspath))
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, file, compress=compress)
for features_dir in features_dirs:
finder = FileFinder(features_dir, find_executables=False)
for p, f in finder.find('**'):
add(mozpath.join('assets', 'features', p), f, False)
for assets_dir in assets_dirs:
finder = FileFinder(assets_dir, find_executables=False)
for p, f in finder.find('**'):
compress = None # Take default from Jarrer.
if p.endswith('.so'):
# Asset libraries are special.
if f.open().read(5)[1:] == '7zXZ':
print('%s is already compressed' % p)
# We need to store (rather than deflate) compressed libraries
# (even if we don't compress them ourselves).
compress = False
elif buildconfig.substs.get('XZ'):
cmd = [
buildconfig.substs.get('XZ'), '-zkf',
mozpath.join(finder.base, p)
]
bcj = None
if buildconfig.substs.get('MOZ_THUMB2'):
bcj = '--armthumb'
elif buildconfig.substs.get('CPU_ARCH') == 'arm':
bcj = '--arm'
elif buildconfig.substs.get('CPU_ARCH') == 'x86':
bcj = '--x86'
if bcj:
cmd.extend([bcj, '--lzma2'])
print('xz-compressing %s with %s' % (p, ' '.join(cmd)))
subprocess.check_output(cmd)
os.rename(f.path + '.xz', f.path)
compress = False
add(mozpath.join('assets', p), f, compress=compress)
for lib_dir in lib_dirs:
finder = FileFinder(lib_dir, find_executables=False)
for p, f in finder.find('**'):
add(mozpath.join('lib', p), f)
for root_file in root_files:
add(os.path.basename(root_file), File(root_file))
if omni_ja:
add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False)
if classes_dex:
add('classes.dex', File(classes_dex))
return jarrer
def package_fennec_apk(inputs=[], omni_ja=None, classes_dex=None,
lib_dirs=[],
assets_dirs=[],
features_dirs=[],
root_files=[],
verbose=False):
jarrer = Jarrer(optimize=False)
# First, take input files. The contents of the later files overwrites the
# content of earlier files. Multidexing requires special care: we want a
# coherent set of classesN.dex files, so we only take DEX files from a
# single input. This avoids taking, say, classes{1,2,3}.dex from the first
# input and only classes{1,2}.dex from the second input, leading to
# (potentially) duplicated symbols at runtime.
last_input_with_dex_files = None
for input in inputs:
jar = JarReader(input)
for file in jar:
path = file.filename
if mozpath.match(path, '/classes*.dex'):
last_input_with_dex_files = input
continue
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, DeflatedFile(file), compress=file.compressed)
# If we have an input with DEX files, take them all here.
if last_input_with_dex_files:
jar = JarReader(last_input_with_dex_files)
for file in jar:
path = file.filename
if not mozpath.match(path, '/classes*.dex'):
continue
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, DeflatedFile(file), compress=file.compressed)
def add(path, file, compress=None):
abspath = os.path.abspath(file.path)
if verbose:
print('Packaging %s from %s' % (path, file.path))
if not os.path.exists(abspath):
raise ValueError('File %s not found (looked for %s)' % \
(file.path, abspath))
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, file, compress=compress)
for features_dir in features_dirs:
finder = FileFinder(features_dir)
for p, f in finder.find('**'):
add(mozpath.join('assets', 'features', p), f, False)
for assets_dir in assets_dirs:
finder = FileFinder(assets_dir)
for p, f in finder.find('**'):
compress = None # Take default from Jarrer.
if p.endswith('.so'):
# Asset libraries are special.
if f.open().read(5)[1:] == '7zXZ':
print('%s is already compressed' % p)
# We need to store (rather than deflate) compressed libraries
# (even if we don't compress them ourselves).
compress = False
elif buildconfig.substs.get('XZ'):
cmd = [buildconfig.substs.get('XZ'), '-zkf',
mozpath.join(finder.base, p)]
# For now, the mozglue XZStream ELF loader can only support xz files
# with a single stream that contains a single block. In xz, there is no
# explicit option to set the max block count. Instead, we force xz to use
# single thread mode, which results in a single block.
cmd.extend(['--threads=1'])
bcj = None
if buildconfig.substs.get('MOZ_THUMB2'):
bcj = '--armthumb'
elif buildconfig.substs.get('CPU_ARCH') == 'arm':
bcj = '--arm'
elif buildconfig.substs.get('CPU_ARCH') == 'x86':
bcj = '--x86'
if bcj:
cmd.extend([bcj])
# We need to explicitly specify the LZMA filter chain to ensure consistent builds
# across platforms. Note that the dict size must be less then 16MiB per the hardcoded
# value in mozglue/linker/XZStream.cpp. This is the default LZMA filter chain for for
# xz-utils version 5.0. See:
# https://github.com/xz-mirror/xz/blob/v5.0.0/src/liblzma/lzma/lzma_encoder_presets.c
# https://github.com/xz-mirror/xz/blob/v5.0.0/src/liblzma/api/lzma/container.h#L31
cmd.extend(['--lzma2=dict=8MiB,lc=3,lp=0,pb=2,mode=normal,nice=64,mf=bt4,depth=0'])
print('xz-compressing %s with %s' % (p, ' '.join(cmd)))
subprocess.check_output(cmd)
os.rename(f.path + '.xz', f.path)
compress = False
add(mozpath.join('assets', p), f, compress=compress)
for lib_dir in lib_dirs:
finder = FileFinder(lib_dir)
for p, f in finder.find('**'):
add(mozpath.join('lib', p), f)
for root_file in root_files:
add(os.path.basename(root_file), File(root_file))
if omni_ja:
add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False)
if classes_dex:
if buildconfig.substs.get('MOZ_BUILD_MOBILE_ANDROID_WITH_GRADLE'):
raise ValueError("Fennec APKs built --with-gradle "
"should never specify classes.dex")
add('classes.dex', File(classes_dex))
return jarrer
def test_jarrer(self):
copier = Jarrer()
copier.add('foo/bar', GeneratedFile('foobar'))
copier.add('foo/qux', GeneratedFile('fooqux'))
copier.add('foo/deep/nested/directory/file', GeneratedFile('fooz'))
copier.add('bar', GeneratedFile('bar'))
copier.add('qux/foo', GeneratedFile('quxfoo'))
copier.add('qux/bar', GeneratedFile(''))
dest = MockDest()
copier.copy(dest)
self.check_jar(dest, copier)
copier.remove('foo')
copier.add('test', GeneratedFile('test'))
copier.copy(dest)
self.check_jar(dest, copier)
copier.remove('test')
copier.add('test', GeneratedFile('replaced-content'))
copier.copy(dest)
self.check_jar(dest, copier)
copier.copy(dest)
self.check_jar(dest, copier)
preloaded = ['qux/bar', 'bar']
copier.preload(preloaded)
copier.copy(dest)
dest.seek(0)
jar = JarReader(fileobj=dest)
self.assertEqual([f.filename for f in jar], preloaded +
[p for p in copier.paths() if not p in preloaded])
self.assertEqual(jar.last_preloaded, preloaded[-1])
def test_jarrer(self):
copier = Jarrer()
copier.add("foo/bar", GeneratedFile(b"foobar"))
copier.add("foo/qux", GeneratedFile(b"fooqux"))
copier.add("foo/deep/nested/directory/file", GeneratedFile(b"fooz"))
copier.add("bar", GeneratedFile(b"bar"))
copier.add("qux/foo", GeneratedFile(b"quxfoo"))
copier.add("qux/bar", GeneratedFile(b""))
dest = MockDest()
copier.copy(dest)
self.check_jar(dest, copier)
copier.remove("foo")
copier.add("test", GeneratedFile(b"test"))
copier.copy(dest)
self.check_jar(dest, copier)
copier.remove("test")
copier.add("test", GeneratedFile(b"replaced-content"))
copier.copy(dest)
self.check_jar(dest, copier)
copier.copy(dest)
self.check_jar(dest, copier)
preloaded = ["qux/bar", "bar"]
copier.preload(preloaded)
copier.copy(dest)
dest.seek(0)
jar = JarReader(fileobj=dest)
self.assertEqual(
[f.filename for f in jar],
preloaded + [p for p in copier.paths() if p not in preloaded],
)
self.assertEqual(jar.last_preloaded, preloaded[-1])
def test_jarrer(self):
copier = Jarrer()
copier.add('foo/bar', GeneratedFile('foobar'))
copier.add('foo/qux', GeneratedFile('fooqux'))
copier.add('foo/deep/nested/directory/file', GeneratedFile('fooz'))
copier.add('bar', GeneratedFile('bar'))
copier.add('qux/foo', GeneratedFile('quxfoo'))
copier.add('qux/bar', GeneratedFile(''))
dest = MockDest()
copier.copy(dest)
self.check_jar(dest, copier)
copier.remove('foo')
copier.add('test', GeneratedFile('test'))
copier.copy(dest)
self.check_jar(dest, copier)
copier.remove('test')
copier.add('test', GeneratedFile('replaced-content'))
copier.copy(dest)
self.check_jar(dest, copier)
copier.copy(dest)
self.check_jar(dest, copier)
preloaded = ['qux/bar', 'bar']
copier.preload(preloaded)
copier.copy(dest)
dest.seek(0)
jar = JarReader(fileobj=dest)
self.assertEqual([f.filename for f in jar], preloaded +
[p for p in copier.paths() if not p in preloaded])
self.assertEqual(jar.last_preloaded, preloaded[-1])
def package_geckoview_aar(topsrcdir, distdir, appname, output_file):
jarrer = Jarrer(optimize=False)
app_path = os.path.join(distdir, appname)
assets = FileFinder(os.path.join(app_path, "assets"), ignore=["*.so"])
for p, f in assets.find("omni.ja"):
jarrer.add(os.path.join("assets", p), f)
# The folder that contains Fennec's JAR files and resources.
base_path = os.path.join(distdir, "..", "mobile", "android", "base")
# The resource set is packaged during Fennec's build.
resjar = JarReader(os.path.join(base_path, "geckoview_resources.zip"))
for p, f in JarFinder(base_path, resjar).find("*"):
jarrer.add(os.path.join("res", p), f)
# Package the contents of all Fennec JAR files into classes.jar.
classes_jar_file = _generate_geckoview_classes_jar(distdir, base_path)
jarrer.add("classes.jar", classes_jar_file)
# Add R.txt.
jarrer.add("R.txt", File(os.path.join(base_path, "R.txt")))
# Finally add AndroidManifest.xml.
srcdir = os.path.join(topsrcdir, "mobile", "android", "geckoview_library", "geckoview")
jarrer.add("AndroidManifest.xml", File(os.path.join(srcdir, "AndroidManifest.xml")))
jarrer.copy(output_file)
return 0
def package_geckolibs_aar(topsrcdir, distdir, appname, output_file):
jarrer = Jarrer(optimize=False)
srcdir = os.path.join(topsrcdir, 'mobile', 'android', 'geckoview_library', 'geckolibs')
jarrer.add('AndroidManifest.xml', File(os.path.join(srcdir, 'AndroidManifest.xml')))
jarrer.add('classes.jar', File(os.path.join(srcdir, 'classes.jar')))
jni = FileFinder(os.path.join(distdir, appname, 'lib'))
for p, f in jni.find('**/*.so'):
jarrer.add(os.path.join('jni', p), f)
# Include the buildinfo JSON as an asset, to give future consumers at least
# a hope of determining where this AAR came from.
json = FileFinder(distdir, ignore=['*.mozinfo.json'])
for p, f in json.find('*.json'):
jarrer.add(os.path.join('assets', p), f)
# This neatly ignores omni.ja.
assets = FileFinder(os.path.join(distdir, appname, 'assets'))
for p, f in assets.find('**/*.so'):
jarrer.add(os.path.join('assets', p), f)
jarrer.copy(output_file)
return 0
def package_geckolibs_aar(topsrcdir, distdir, appname, output_file):
jarrer = Jarrer(optimize=False)
srcdir = os.path.join(topsrcdir, "mobile", "android", "geckoview_library", "geckolibs")
jarrer.add("AndroidManifest.xml", File(os.path.join(srcdir, "AndroidManifest.xml")))
jarrer.add("classes.jar", File(os.path.join(srcdir, "classes.jar")))
jni = FileFinder(os.path.join(distdir, appname, "lib"))
for p, f in jni.find("**/*.so"):
jarrer.add(os.path.join("jni", p), f)
# Include the buildinfo JSON as an asset, to give future consumers at least
# a hope of determining where this AAR came from.
json = FileFinder(distdir, ignore=["*.mozinfo.json"])
for p, f in json.find("*.json"):
jarrer.add(os.path.join("assets", p), f)
# This neatly ignores omni.ja.
assets = FileFinder(os.path.join(distdir, appname, "assets"))
for p, f in assets.find("**/*.so"):
jarrer.add(os.path.join("assets", p), f)
jarrer.copy(output_file)
return 0
def package_fennec_apk(inputs=[],
omni_ja=None,
classes_dex=None,
lib_dirs=[],
assets_dirs=[],
features_dirs=[],
root_files=[],
verbose=False):
jarrer = Jarrer(optimize=False)
# First, take input files. The contents of the later files overwrites the
# content of earlier files. Multidexing requires special care: we want a
# coherent set of classesN.dex files, so we only take DEX files from a
# single input. This avoids taking, say, classes{1,2,3}.dex from the first
# input and only classes{1,2}.dex from the second input, leading to
# (potentially) duplicated symbols at runtime.
last_input_with_dex_files = None
for input in inputs:
jar = JarReader(input)
for file in jar:
path = file.filename
if mozpath.match(path, '/classes*.dex'):
last_input_with_dex_files = input
continue
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, DeflatedFile(file), compress=file.compressed)
# If we have an input with DEX files, take them all here.
if last_input_with_dex_files:
jar = JarReader(last_input_with_dex_files)
for file in jar:
path = file.filename
if not mozpath.match(path, '/classes*.dex'):
continue
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, DeflatedFile(file), compress=file.compressed)
def add(path, file, compress=None):
abspath = os.path.abspath(file.path)
if verbose:
print('Packaging %s from %s' % (path, file.path))
if not os.path.exists(abspath):
raise ValueError('File %s not found (looked for %s)' % \
(file.path, abspath))
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, file, compress=compress)
for features_dir in features_dirs:
finder = FileFinder(features_dir)
for p, f in finder.find('**'):
add(mozpath.join('assets', 'features', p), f, False)
for assets_dir in assets_dirs:
finder = FileFinder(assets_dir)
for p, f in finder.find('**'):
compress = None # Take default from Jarrer.
if p.endswith('.so'):
# Asset libraries are special.
if f.open().read(5)[1:] == '7zXZ':
print('%s is already compressed' % p)
# We need to store (rather than deflate) compressed libraries
# (even if we don't compress them ourselves).
compress = False
elif buildconfig.substs.get('XZ'):
cmd = [
buildconfig.substs.get('XZ'), '-zkf',
mozpath.join(finder.base, p)
]
# For now, the mozglue XZStream ELF loader can only support xz files
# with a single stream that contains a single block. In xz, there is no
# explicit option to set the max block count. Instead, we force xz to use
# single thread mode, which results in a single block.
cmd.extend(['--threads=1'])
bcj = None
if buildconfig.substs.get('MOZ_THUMB2'):
bcj = '--armthumb'
elif buildconfig.substs.get('CPU_ARCH') == 'arm':
bcj = '--arm'
elif buildconfig.substs.get('CPU_ARCH') == 'x86':
bcj = '--x86'
if bcj:
cmd.extend([bcj])
# We need to explicitly specify the LZMA filter chain to ensure consistent builds
# across platforms. Note that the dict size must be less then 16MiB per the hardcoded
# value in mozglue/linker/XZStream.cpp. This is the default LZMA filter chain for for
# xz-utils version 5.0. See:
# https://github.com/xz-mirror/xz/blob/v5.0.0/src/liblzma/lzma/lzma_encoder_presets.c
# https://github.com/xz-mirror/xz/blob/v5.0.0/src/liblzma/api/lzma/container.h#L31
cmd.extend([
'--lzma2=dict=8MiB,lc=3,lp=0,pb=2,mode=normal,nice=64,mf=bt4,depth=0'
])
print('xz-compressing %s with %s' % (p, ' '.join(cmd)))
subprocess.check_output(cmd)
os.rename(f.path + '.xz', f.path)
compress = False
add(mozpath.join('assets', p), f, compress=compress)
for lib_dir in lib_dirs:
finder = FileFinder(lib_dir)
for p, f in finder.find('**'):
add(mozpath.join('lib', p), f)
for root_file in root_files:
add(os.path.basename(root_file), File(root_file))
if omni_ja:
add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False)
if classes_dex:
if buildconfig.substs.get('MOZ_BUILD_MOBILE_ANDROID_WITH_GRADLE'):
raise ValueError("Fennec APKs built --with-gradle "
"should never specify classes.dex")
add('classes.dex', File(classes_dex))
return jarrer
def package_geckoview_aar(topsrcdir, distdir, appname, output_file):
jarrer = Jarrer(optimize=False)
app_path = os.path.join(distdir, appname)
assets = FileFinder(os.path.join(app_path, 'assets'), ignore=['*.so'])
for p, f in assets.find('omni.ja'):
jarrer.add(os.path.join('assets', p), f)
# The folder that contains Fennec's JAR files and resources.
base_path = os.path.join(distdir, '..', 'mobile', 'android', 'base')
# The resource set is packaged during Fennec's build.
resjar = JarReader(os.path.join(base_path, 'geckoview_resources.zip'))
for p, f in JarFinder(base_path, resjar).find('*'):
jarrer.add(os.path.join('res', p), f)
# Package the contents of all Fennec JAR files into classes.jar.
classes_jar_file = _generate_geckoview_classes_jar(distdir, base_path)
jarrer.add('classes.jar', classes_jar_file)
# Add R.txt.
jarrer.add('R.txt', File(os.path.join(base_path, 'R.txt')))
# Finally add AndroidManifest.xml.
srcdir = os.path.join(topsrcdir, 'mobile', 'android', 'geckoview_library', 'geckoview')
jarrer.add('AndroidManifest.xml', File(os.path.join(srcdir, 'AndroidManifest.xml')))
jarrer.copy(output_file)
return 0
def package_fennec_apk(inputs=[], omni_ja=None, classes_dex=None,
lib_dirs=[],
assets_dirs=[],
szip_assets_libs_with=None,
root_files=[],
verbose=False):
jarrer = Jarrer(optimize=False)
# First, take input files. The contents of the later files overwrites the
# content of earlier files.
for input in inputs:
jar = JarReader(input)
for file in jar:
path = file.filename
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, DeflatedFile(file), compress=file.compressed)
def add(path, file, compress=None):
abspath = os.path.abspath(file.path)
if verbose:
print('Packaging %s from %s' % (path, file.path))
if not os.path.exists(abspath):
raise ValueError('File %s not found (looked for %s)' % \
(file.path, abspath))
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, file, compress=compress)
for assets_dir in assets_dirs:
finder = FileFinder(assets_dir, find_executables=False)
for p, f in finder.find('**'):
compress = None # Take default from Jarrer.
if p.endswith('.so'):
# Asset libraries are special.
if szip_assets_libs_with:
# We need to szip libraries before packing. The file
# returned by the finder is not yet opened. When it is
# opened, it will "see" the content updated by szip.
subprocess.check_output([szip_assets_libs_with,
mozpath.join(finder.base, p)])
if f.open().read(4) == 'SeZz':
# We need to store (rather than deflate) szipped libraries
# (even if we don't szip them ourselves).
compress = False
add(mozpath.join('assets', p), f, compress=compress)
for lib_dir in lib_dirs:
finder = FileFinder(lib_dir, find_executables=False)
for p, f in finder.find('**'):
add(mozpath.join('lib', p), f)
for root_file in root_files:
add(os.path.basename(root_file), File(root_file))
if omni_ja:
add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False)
if classes_dex:
add('classes.dex', File(classes_dex))
return jarrer
def package_geckolibs_aar(topsrcdir, distdir, output_file):
jarrer = Jarrer(optimize=False)
srcdir = os.path.join(topsrcdir, 'mobile', 'android', 'geckoview_library',
'geckolibs')
jarrer.add('AndroidManifest.xml',
File(os.path.join(srcdir, 'AndroidManifest.xml')))
jarrer.add('classes.jar', File(os.path.join(srcdir, 'classes.jar')))
jni = FileFinder(os.path.join(distdir, 'fennec', 'lib'))
for p, f in jni.find('**/*.so'):
jarrer.add(os.path.join('jni', p), f)
# Include the buildinfo JSON as an asset, to give future consumers at least
# a hope of determining where this AAR came from.
json = FileFinder(distdir, ignore=['*.mozinfo.json'])
for p, f in json.find('*.json'):
jarrer.add(os.path.join('assets', p), f)
# This neatly ignores omni.ja.
assets = FileFinder(os.path.join(distdir, 'fennec', 'assets'))
for p, f in assets.find('**/*.so'):
jarrer.add(os.path.join('assets', p), f)
jarrer.copy(output_file)
return 0
def package_geckoview_aar(topsrcdir, distdir, output_file):
jarrer = Jarrer(optimize=False)
fennec_path = os.path.join(distdir, 'fennec')
assets = FileFinder(os.path.join(fennec_path, 'assets'), ignore=['*.so'])
for p, f in assets.find('omni.ja'):
jarrer.add(os.path.join('assets', p), f)
# The folder that contains Fennec's JAR files and resources.
base_path = os.path.join(distdir, '..', 'mobile', 'android', 'base')
# The resource set is packaged during Fennec's build.
resjar = JarReader(os.path.join(base_path, 'geckoview_resources.zip'))
for p, f in JarFinder(p, resjar).find('*'):
jarrer.add(os.path.join('res', p), f)
# Package the contents of all Fennec JAR files into classes.jar.
classes_jar_file = _generate_geckoview_classes_jar(distdir, base_path)
jarrer.add('classes.jar', classes_jar_file)
# Add R.txt.
jarrer.add('R.txt', File(os.path.join(base_path, 'R.txt')))
# Finally add AndroidManifest.xml.
srcdir = os.path.join(topsrcdir, 'mobile', 'android', 'geckoview_library',
'geckoview')
jarrer.add('AndroidManifest.xml',
File(os.path.join(srcdir, 'AndroidManifest.xml')))
jarrer.copy(output_file)
return 0
def package_fennec_apk(inputs=[],
omni_ja=None,
lib_dirs=[],
assets_dirs=[],
features_dirs=[],
root_files=[],
verbose=False):
jarrer = Jarrer()
# First, take input files. The contents of the later files overwrites the
# content of earlier files. Multidexing requires special care: we want a
# coherent set of classesN.dex files, so we only take DEX files from a
# single input. This avoids taking, say, classes{1,2,3}.dex from the first
# input and only classes{1,2}.dex from the second input, leading to
# (potentially) duplicated symbols at runtime.
last_input_with_dex_files = None
for input in inputs:
jar = JarReader(input)
for file in jar:
path = file.filename
if mozpath.match(path, '/classes*.dex'):
last_input_with_dex_files = input
continue
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, DeflatedFile(file), compress=file.compressed)
# If we have an input with DEX files, take them all here.
if last_input_with_dex_files:
jar = JarReader(last_input_with_dex_files)
for file in jar:
path = file.filename
if not mozpath.match(path, '/classes*.dex'):
continue
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, DeflatedFile(file), compress=file.compressed)
def add(path, file, compress=None):
abspath = os.path.abspath(file.path)
if verbose:
print('Packaging %s from %s' % (path, file.path))
if not os.path.exists(abspath):
raise ValueError('File %s not found (looked for %s)' %
(file.path, abspath))
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, file, compress=compress)
for features_dir in features_dirs:
finder = FileFinder(features_dir)
for p, f in finder.find('**'):
add(mozpath.join('assets', 'features', p), f, False)
for assets_dir in assets_dirs:
finder = FileFinder(assets_dir)
for p, f in finder.find('**'):
add(mozpath.join('assets', p), f)
for lib_dir in lib_dirs:
finder = FileFinder(lib_dir)
for p, f in finder.find('**'):
add(mozpath.join('lib', p), f)
for root_file in root_files:
add(os.path.basename(root_file), File(root_file))
if omni_ja:
add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False)
return jarrer
def test_jarrer(self):
copier = Jarrer()
copier.add("foo/bar", GeneratedFile("foobar"))
copier.add("foo/qux", GeneratedFile("fooqux"))
copier.add("foo/deep/nested/directory/file", GeneratedFile("fooz"))
copier.add("bar", GeneratedFile("bar"))
copier.add("qux/foo", GeneratedFile("quxfoo"))
copier.add("qux/bar", GeneratedFile(""))
dest = MockDest()
copier.copy(dest)
self.check_jar(dest, copier)
copier.remove("foo")
copier.add("test", GeneratedFile("test"))
copier.copy(dest)
self.check_jar(dest, copier)
copier.remove("test")
copier.add("test", GeneratedFile("replaced-content"))
copier.copy(dest)
self.check_jar(dest, copier)
copier.copy(dest)
self.check_jar(dest, copier)
preloaded = ["qux/bar", "bar"]
copier.preload(preloaded)
copier.copy(dest)
dest.seek(0)
jar = JarReader(fileobj=dest)
self.assertEqual([f.filename for f in jar], preloaded + [p for p in copier.paths() if not p in preloaded])
self.assertEqual(jar.last_preloaded, preloaded[-1])
def package_fennec_apk(inputs=[],
omni_ja=None,
classes_dex=None,
lib_dirs=[],
assets_dirs=[],
szip_assets_libs_with=None,
root_files=[],
verbose=False):
jarrer = Jarrer(optimize=False)
# First, take input files. The contents of the later files overwrites the
# content of earlier files.
for input in inputs:
jar = JarReader(input)
for file in jar:
path = file.filename
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, DeflatedFile(file), compress=file.compressed)
def add(path, file, compress=None):
abspath = os.path.abspath(file.path)
if verbose:
print('Packaging %s from %s' % (path, file.path))
if not os.path.exists(abspath):
raise ValueError('File %s not found (looked for %s)' % \
(file.path, abspath))
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, file, compress=compress)
for assets_dir in assets_dirs:
finder = FileFinder(assets_dir, find_executables=False)
for p, f in finder.find('**'):
compress = None # Take default from Jarrer.
if p.endswith('.so'):
# Asset libraries are special.
if szip_assets_libs_with:
# We need to szip libraries before packing. The file
# returned by the finder is not yet opened. When it is
# opened, it will "see" the content updated by szip.
subprocess.check_output(
[szip_assets_libs_with,
mozpath.join(finder.base, p)])
if f.open().read(4) == 'SeZz':
# We need to store (rather than deflate) szipped libraries
# (even if we don't szip them ourselves).
compress = False
add(mozpath.join('assets', p), f, compress=compress)
for lib_dir in lib_dirs:
finder = FileFinder(lib_dir, find_executables=False)
for p, f in finder.find('**'):
add(mozpath.join('lib', p), f)
for root_file in root_files:
add(os.path.basename(root_file), File(root_file))
if omni_ja:
add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False)
if classes_dex:
add('classes.dex', File(classes_dex))
return jarrer
def package_fennec_apk(inputs=[], omni_ja=None, classes_dex=None,
lib_dirs=[],
assets_dirs=[],
features_dirs=[],
root_files=[],
verbose=False):
jarrer = Jarrer(optimize=False)
# First, take input files. The contents of the later files overwrites the
# content of earlier files.
for input in inputs:
jar = JarReader(input)
for file in jar:
path = file.filename
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, DeflatedFile(file), compress=file.compressed)
def add(path, file, compress=None):
abspath = os.path.abspath(file.path)
if verbose:
print('Packaging %s from %s' % (path, file.path))
if not os.path.exists(abspath):
raise ValueError('File %s not found (looked for %s)' % \
(file.path, abspath))
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, file, compress=compress)
for features_dir in features_dirs:
finder = FileFinder(features_dir)
for p, f in finder.find('**'):
add(mozpath.join('assets', 'features', p), f, False)
for assets_dir in assets_dirs:
finder = FileFinder(assets_dir)
for p, f in finder.find('**'):
compress = None # Take default from Jarrer.
if p.endswith('.so'):
# Asset libraries are special.
if f.open().read(5)[1:] == '7zXZ':
print('%s is already compressed' % p)
# We need to store (rather than deflate) compressed libraries
# (even if we don't compress them ourselves).
compress = False
elif buildconfig.substs.get('XZ'):
cmd = [buildconfig.substs.get('XZ'), '-zkf',
mozpath.join(finder.base, p)]
bcj = None
if buildconfig.substs.get('MOZ_THUMB2'):
bcj = '--armthumb'
elif buildconfig.substs.get('CPU_ARCH') == 'arm':
bcj = '--arm'
elif buildconfig.substs.get('CPU_ARCH') == 'x86':
bcj = '--x86'
if bcj:
cmd.extend([bcj, '--lzma2'])
print('xz-compressing %s with %s' % (p, ' '.join(cmd)))
subprocess.check_output(cmd)
os.rename(f.path + '.xz', f.path)
compress = False
add(mozpath.join('assets', p), f, compress=compress)
for lib_dir in lib_dirs:
finder = FileFinder(lib_dir)
for p, f in finder.find('**'):
add(mozpath.join('lib', p), f)
for root_file in root_files:
add(os.path.basename(root_file), File(root_file))
if omni_ja:
add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False)
if classes_dex:
add('classes.dex', File(classes_dex))
return jarrer
