def test_deflated_file_no_write(self):
"""
Test various conditions where DeflatedFile.copy is expected not to
write in the destination file.
"""
src = self.tmppath("src.jar")
dest = self.tmppath("dest")
with JarWriter(src) as jar:
jar.add("test", "test")
jar.add("test2", "test")
jar.add("fooo", "fooo")
jar = JarReader(src)
# Initial copy
f = DeflatedFile(jar["test"])
f.copy(dest)
# Ensure subsequent copies won't trigger writes
f.copy(DestNoWrite(dest))
self.assertEqual("test", open(dest, "rb").read())
# When using a different file with the same content, no copy should
# occur
f = DeflatedFile(jar["test2"])
f.copy(DestNoWrite(dest))
self.assertEqual("test", open(dest, "rb").read())
# Double check that under conditions where a copy occurs, we would get
# an exception.
f = DeflatedFile(jar["fooo"])
self.assertRaises(RuntimeError, f.copy, DestNoWrite(dest))
def test_deflated_file_no_write(self):
'''
Test various conditions where DeflatedFile.copy is expected not to
write in the destination file.
'''
src = self.tmppath('src.jar')
dest = self.tmppath('dest')
with JarWriter(src) as jar:
jar.add('test', 'test')
jar.add('test2', 'test')
jar.add('fooo', 'fooo')
jar = JarReader(src)
# Initial copy
f = DeflatedFile(jar['test'])
f.copy(dest)
# Ensure subsequent copies won't trigger writes
f.copy(DestNoWrite(dest))
self.assertEqual('test', open(dest, 'rb').read())
# When using a different file with the same content, no copy should
# occur
f = DeflatedFile(jar['test2'])
f.copy(DestNoWrite(dest))
self.assertEqual('test', open(dest, 'rb').read())
# Double check that under conditions where a copy occurs, we would get
# an exception.
f = DeflatedFile(jar['fooo'])
self.assertRaises(RuntimeError, f.copy, DestNoWrite(dest))
def test_deflated_file_open(self):
'''
Test whether DeflatedFile.open returns an appropriately reset file
object.
'''
src = self.tmppath('src.jar')
content = b''.join(samples)
with JarWriter(src) as jar:
jar.add('content', content)
f = DeflatedFile(JarReader(src)['content'])
self.assertEqual(content[:42], f.open().read(42))
self.assertEqual(content, f.open().read())
def test_deflated_file_open(self):
'''
Test whether DeflatedFile.open returns an appropriately reset file
object.
'''
src = self.tmppath('src.jar')
content = ''.join(samples)
with JarWriter(src) as jar:
jar.add('content', content)
f = DeflatedFile(JarReader(src)['content'])
self.assertEqual(content[:42], f.open().read(42))
self.assertEqual(content, f.open().read())
def test_deflated_file_open(self):
"""
Test whether DeflatedFile.open returns an appropriately reset file
object.
"""
src = self.tmppath("src.jar")
content = "".join(samples)
with JarWriter(src) as jar:
jar.add("content", content)
f = DeflatedFile(JarReader(src)["content"])
self.assertEqual(content[:42], f.open().read(42))
self.assertEqual(content, f.open().read())
def test_deflated_file_open(self):
"""
Test whether DeflatedFile.open returns an appropriately reset file
object.
"""
src = self.tmppath("src.jar")
content = b"".join(samples)
with JarWriter(src) as jar:
jar.add("content", content)
f = DeflatedFile(JarReader(src)["content"])
self.assertEqual(content[:42], f.open().read(42))
self.assertEqual(content, f.open().read())
def _handle_manifest_entry(self, entry, jars):
jarpath = None
if isinstance(entry, ManifestEntryWithRelPath) and \
urlparse(entry.relpath).scheme == 'jar':
jarpath, entry = self._unjarize(entry, entry.relpath)
elif isinstance(entry, ManifestResource) and \
urlparse(entry.target).scheme == 'jar':
jarpath, entry = self._unjarize(entry, entry.target)
if jarpath:
# Don't defer unpacking the jar file. If we already saw
# it, take (and remove) it from the registry. If we
# haven't, try to find it now.
if self.files.contains(jarpath):
jar = self.files[jarpath]
self.files.remove(jarpath)
else:
jar = [f for p, f in FileFinder.find(self, jarpath)]
assert len(jar) == 1
jar = jar[0]
if not jarpath in jars:
base = mozpack.path.splitext(jarpath)[0]
for j in self._open_jar(jarpath, jar):
self.files.add(mozpack.path.join(base, j.filename),
DeflatedFile(j))
jars.add(jarpath)
self.kind = 'jar'
return entry
def test_deflated_file(self):
"""
Check that DeflatedFile.copy yields the proper content in the
destination file in all situations that trigger different code paths
(see TestFile.test_file)
"""
src = self.tmppath("src.jar")
dest = self.tmppath("dest")
contents = {}
with JarWriter(src) as jar:
for content in samples:
name = "".join(random.choice(string.letters) for i in xrange(8))
jar.add(name, content, compress=True)
contents[name] = content
for j in JarReader(src):
f = DeflatedFile(j)
f.copy(dest)
self.assertEqual(contents[j.filename], open(dest, "rb").read())
def test_deflated_file(self):
'''
Check that DeflatedFile.copy yields the proper content in the
destination file in all situations that trigger different code paths
(see TestFile.test_file)
'''
src = self.tmppath('src.jar')
dest = self.tmppath('dest')
contents = {}
with JarWriter(src) as jar:
for content in samples:
name = ''.join(
random.choice(string.letters) for i in xrange(8))
jar.add(name, content, compress=True)
contents[name] = content
for j in JarReader(src):
f = DeflatedFile(j)
f.copy(dest)
self.assertEqual(contents[j.filename], open(dest, 'rb').read())
def _fill_with_omnijar(self, base, jar):
for j in jar:
path = mozpack.path.join(base, j.filename)
if is_manifest(j.filename):
m = self.files[path] if self.files.contains(path) \
else ManifestFile(mozpack.path.dirname(path))
for e in parse_manifest(None, path, j):
m.add(e)
if not self.files.contains(path):
self.files.add(path, m)
continue
else:
self.files.add(path, DeflatedFile(j))
def test_deflated_file(self):
"""
Check that DeflatedFile.copy yields the proper content in the
destination file in all situations that trigger different code paths
(see TestFile.test_file)
"""
src = self.tmppath("src.jar")
dest = self.tmppath("dest")
contents = {}
with JarWriter(src) as jar:
for content in samples:
name = "".join(
random.choice(
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
for i in range(8))
jar.add(name, content, compress=True)
contents[name] = content
for j in JarReader(src):
f = DeflatedFile(j)
f.copy(dest)
self.assertEqual(contents[j.filename], open(dest, "rb").read())
def test_deflated_file_no_write(self):
'''
Test various conditions where DeflatedFile.copy is expected not to
write in the destination file.
'''
src = self.tmppath('src.jar')
dest = self.tmppath('dest')
with JarWriter(src) as jar:
jar.add('test', 'test')
jar.add('test2', 'test')
jar.add('fooo', 'fooo')
jar = JarReader(src)
# Initial copy
f = DeflatedFile(jar['test'])
f.copy(dest)
# Ensure subsequent copies won't trigger writes
f.copy(DestNoWrite(dest))
self.assertEqual('test', open(dest, 'rb').read())
# When using a different file with the same content, no copy should
# occur
f = DeflatedFile(jar['test2'])
f.copy(DestNoWrite(dest))
self.assertEqual('test', open(dest, 'rb').read())
# Double check that under conditions where a copy occurs, we would get
# an exception.
f = DeflatedFile(jar['fooo'])
self.assertRaises(RuntimeError, f.copy, DestNoWrite(dest))
def package_fennec_apk(inputs=[],
omni_ja=None,
classes_dex=None,
lib_dirs=[],
assets_dirs=[],
szip_assets_libs_with=None,
root_files=[],
verbose=False):
jarrer = Jarrer(optimize=False)
# First, take input files. The contents of the later files overwrites the
# content of earlier files.
for input in inputs:
jar = JarReader(input)
for file in jar:
path = file.filename
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, DeflatedFile(file), compress=file.compressed)
def add(path, file, compress=None):
abspath = os.path.abspath(file.path)
if verbose:
print('Packaging %s from %s' % (path, file.path))
if not os.path.exists(abspath):
raise ValueError('File %s not found (looked for %s)' % \
(file.path, abspath))
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, file, compress=compress)
for assets_dir in assets_dirs:
finder = FileFinder(assets_dir, find_executables=False)
for p, f in finder.find('**'):
compress = None # Take default from Jarrer.
if p.endswith('.so'):
# Asset libraries are special.
if szip_assets_libs_with:
# We need to szip libraries before packing. The file
# returned by the finder is not yet opened. When it is
# opened, it will "see" the content updated by szip.
subprocess.check_output(
[szip_assets_libs_with,
mozpath.join(finder.base, p)])
if f.open().read(4) == 'SeZz':
# We need to store (rather than deflate) szipped libraries
# (even if we don't szip them ourselves).
compress = False
add(mozpath.join('assets', p), f, compress=compress)
for lib_dir in lib_dirs:
finder = FileFinder(lib_dir, find_executables=False)
for p, f in finder.find('**'):
add(mozpath.join('lib', p), f)
for root_file in root_files:
add(os.path.basename(root_file), File(root_file))
if omni_ja:
add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False)
if classes_dex:
add('classes.dex', File(classes_dex))
return jarrer
def package_fennec_apk(inputs=[],
omni_ja=None,
classes_dex=None,
lib_dirs=[],
assets_dirs=[],
features_dirs=[],
root_files=[],
verbose=False):
jarrer = Jarrer(optimize=False)
# First, take input files. The contents of the later files overwrites the
# content of earlier files. Multidexing requires special care: we want a
# coherent set of classesN.dex files, so we only take DEX files from a
# single input. This avoids taking, say, classes{1,2,3}.dex from the first
# input and only classes{1,2}.dex from the second input, leading to
# (potentially) duplicated symbols at runtime.
last_input_with_dex_files = None
for input in inputs:
jar = JarReader(input)
for file in jar:
path = file.filename
if mozpath.match(path, '/classes*.dex'):
last_input_with_dex_files = input
continue
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, DeflatedFile(file), compress=file.compressed)
# If we have an input with DEX files, take them all here.
if last_input_with_dex_files:
jar = JarReader(last_input_with_dex_files)
for file in jar:
path = file.filename
if not mozpath.match(path, '/classes*.dex'):
continue
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, DeflatedFile(file), compress=file.compressed)
def add(path, file, compress=None):
abspath = os.path.abspath(file.path)
if verbose:
print('Packaging %s from %s' % (path, file.path))
if not os.path.exists(abspath):
raise ValueError('File %s not found (looked for %s)' % \
(file.path, abspath))
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, file, compress=compress)
for features_dir in features_dirs:
finder = FileFinder(features_dir)
for p, f in finder.find('**'):
add(mozpath.join('assets', 'features', p), f, False)
for assets_dir in assets_dirs:
finder = FileFinder(assets_dir)
for p, f in finder.find('**'):
compress = None # Take default from Jarrer.
if p.endswith('.so'):
# Asset libraries are special.
if f.open().read(5)[1:] == '7zXZ':
print('%s is already compressed' % p)
# We need to store (rather than deflate) compressed libraries
# (even if we don't compress them ourselves).
compress = False
elif buildconfig.substs.get('XZ'):
cmd = [
buildconfig.substs.get('XZ'), '-zkf',
mozpath.join(finder.base, p)
]
# For now, the mozglue XZStream ELF loader can only support xz files
# with a single stream that contains a single block. In xz, there is no
# explicit option to set the max block count. Instead, we force xz to use
# single thread mode, which results in a single block.
cmd.extend(['--threads=1'])
bcj = None
if buildconfig.substs.get('MOZ_THUMB2'):
bcj = '--armthumb'
elif buildconfig.substs.get('CPU_ARCH') == 'arm':
bcj = '--arm'
elif buildconfig.substs.get('CPU_ARCH') == 'x86':
bcj = '--x86'
if bcj:
cmd.extend([bcj])
# We need to explicitly specify the LZMA filter chain to ensure consistent builds
# across platforms. Note that the dict size must be less then 16MiB per the hardcoded
# value in mozglue/linker/XZStream.cpp. This is the default LZMA filter chain for for
# xz-utils version 5.0. See:
# https://github.com/xz-mirror/xz/blob/v5.0.0/src/liblzma/lzma/lzma_encoder_presets.c
# https://github.com/xz-mirror/xz/blob/v5.0.0/src/liblzma/api/lzma/container.h#L31
cmd.extend([
'--lzma2=dict=8MiB,lc=3,lp=0,pb=2,mode=normal,nice=64,mf=bt4,depth=0'
])
print('xz-compressing %s with %s' % (p, ' '.join(cmd)))
subprocess.check_output(cmd)
os.rename(f.path + '.xz', f.path)
compress = False
add(mozpath.join('assets', p), f, compress=compress)
for lib_dir in lib_dirs:
finder = FileFinder(lib_dir)
for p, f in finder.find('**'):
add(mozpath.join('lib', p), f)
for root_file in root_files:
add(os.path.basename(root_file), File(root_file))
if omni_ja:
add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False)
if classes_dex:
if buildconfig.substs.get('MOZ_BUILD_MOBILE_ANDROID_WITH_GRADLE'):
raise ValueError("Fennec APKs built --with-gradle "
"should never specify classes.dex")
add('classes.dex', File(classes_dex))
return jarrer
def package_fennec_apk(inputs=[],
omni_ja=None,
classes_dex=None,
lib_dirs=[],
assets_dirs=[],
features_dirs=[],
root_files=[],
verbose=False):
jarrer = Jarrer(optimize=False)
# First, take input files. The contents of the later files overwrites the
# content of earlier files.
for input in inputs:
jar = JarReader(input)
for file in jar:
path = file.filename
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, DeflatedFile(file), compress=file.compressed)
def add(path, file, compress=None):
abspath = os.path.abspath(file.path)
if verbose:
print('Packaging %s from %s' % (path, file.path))
if not os.path.exists(abspath):
raise ValueError('File %s not found (looked for %s)' % \
(file.path, abspath))
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, file, compress=compress)
for features_dir in features_dirs:
finder = FileFinder(features_dir, find_executables=False)
for p, f in finder.find('**'):
add(mozpath.join('assets', 'features', p), f, False)
for assets_dir in assets_dirs:
finder = FileFinder(assets_dir, find_executables=False)
for p, f in finder.find('**'):
compress = None # Take default from Jarrer.
if p.endswith('.so'):
# Asset libraries are special.
if f.open().read(5)[1:] == '7zXZ':
print('%s is already compressed' % p)
# We need to store (rather than deflate) compressed libraries
# (even if we don't compress them ourselves).
compress = False
elif buildconfig.substs.get('XZ'):
cmd = [
buildconfig.substs.get('XZ'), '-zkf',
mozpath.join(finder.base, p)
]
bcj = None
if buildconfig.substs.get('MOZ_THUMB2'):
bcj = '--armthumb'
elif buildconfig.substs.get('CPU_ARCH') == 'arm':
bcj = '--arm'
elif buildconfig.substs.get('CPU_ARCH') == 'x86':
bcj = '--x86'
if bcj:
cmd.extend([bcj, '--lzma2'])
print('xz-compressing %s with %s' % (p, ' '.join(cmd)))
subprocess.check_output(cmd)
os.rename(f.path + '.xz', f.path)
compress = False
add(mozpath.join('assets', p), f, compress=compress)
for lib_dir in lib_dirs:
finder = FileFinder(lib_dir, find_executables=False)
for p, f in finder.find('**'):
add(mozpath.join('lib', p), f)
for root_file in root_files:
add(os.path.basename(root_file), File(root_file))
if omni_ja:
add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False)
if classes_dex:
add('classes.dex', File(classes_dex))
return jarrer
def test_deflated_file_no_write(self):
"""
Test various conditions where DeflatedFile.copy is expected not to
write in the destination file.
"""
src = self.tmppath("src.jar")
dest = self.tmppath("dest")
with JarWriter(src) as jar:
jar.add("test", b"test")
jar.add("test2", b"test")
jar.add("fooo", b"fooo")
jar = JarReader(src)
# Initial copy
f = DeflatedFile(jar["test"])
f.copy(dest)
# Ensure subsequent copies won't trigger writes
f.copy(DestNoWrite(dest))
self.assertEqual(b"test", open(dest, "rb").read())
# When using a different file with the same content, no copy should
# occur
f = DeflatedFile(jar["test2"])
f.copy(DestNoWrite(dest))
self.assertEqual(b"test", open(dest, "rb").read())
# Double check that under conditions where a copy occurs, we would get
# an exception.
f = DeflatedFile(jar["fooo"])
self.assertRaises(RuntimeError, f.copy, DestNoWrite(dest))
def package_fennec_apk(inputs=[],
omni_ja=None,
lib_dirs=[],
assets_dirs=[],
features_dirs=[],
root_files=[],
verbose=False):
jarrer = Jarrer()
# First, take input files. The contents of the later files overwrites the
# content of earlier files. Multidexing requires special care: we want a
# coherent set of classesN.dex files, so we only take DEX files from a
# single input. This avoids taking, say, classes{1,2,3}.dex from the first
# input and only classes{1,2}.dex from the second input, leading to
# (potentially) duplicated symbols at runtime.
last_input_with_dex_files = None
for input in inputs:
jar = JarReader(input)
for file in jar:
path = file.filename
if mozpath.match(path, '/classes*.dex'):
last_input_with_dex_files = input
continue
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, DeflatedFile(file), compress=file.compressed)
# If we have an input with DEX files, take them all here.
if last_input_with_dex_files:
jar = JarReader(last_input_with_dex_files)
for file in jar:
path = file.filename
if not mozpath.match(path, '/classes*.dex'):
continue
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, DeflatedFile(file), compress=file.compressed)
def add(path, file, compress=None):
abspath = os.path.abspath(file.path)
if verbose:
print('Packaging %s from %s' % (path, file.path))
if not os.path.exists(abspath):
raise ValueError('File %s not found (looked for %s)' %
(file.path, abspath))
if jarrer.contains(path):
jarrer.remove(path)
jarrer.add(path, file, compress=compress)
for features_dir in features_dirs:
finder = FileFinder(features_dir)
for p, f in finder.find('**'):
add(mozpath.join('assets', 'features', p), f, False)
for assets_dir in assets_dirs:
finder = FileFinder(assets_dir)
for p, f in finder.find('**'):
add(mozpath.join('assets', p), f)
for lib_dir in lib_dirs:
finder = FileFinder(lib_dir)
for p, f in finder.find('**'):
add(mozpath.join('lib', p), f)
for root_file in root_files:
add(os.path.basename(root_file), File(root_file))
if omni_ja:
add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False)
return jarrer