Esempio n. 1
0
    def test_deflated_file_no_write(self):
        """
        Test various conditions where DeflatedFile.copy is expected not to
        write in the destination file.
        """
        src = self.tmppath("src.jar")
        dest = self.tmppath("dest")

        with JarWriter(src) as jar:
            jar.add("test", "test")
            jar.add("test2", "test")
            jar.add("fooo", "fooo")

        jar = JarReader(src)
        # Initial copy
        f = DeflatedFile(jar["test"])
        f.copy(dest)

        # Ensure subsequent copies won't trigger writes
        f.copy(DestNoWrite(dest))
        self.assertEqual("test", open(dest, "rb").read())

        # When using a different file with the same content, no copy should
        # occur
        f = DeflatedFile(jar["test2"])
        f.copy(DestNoWrite(dest))
        self.assertEqual("test", open(dest, "rb").read())

        # Double check that under conditions where a copy occurs, we would get
        # an exception.
        f = DeflatedFile(jar["fooo"])
        self.assertRaises(RuntimeError, f.copy, DestNoWrite(dest))
Esempio n. 2
0
    def test_deflated_file_no_write(self):
        '''
        Test various conditions where DeflatedFile.copy is expected not to
        write in the destination file.
        '''
        src = self.tmppath('src.jar')
        dest = self.tmppath('dest')

        with JarWriter(src) as jar:
            jar.add('test', 'test')
            jar.add('test2', 'test')
            jar.add('fooo', 'fooo')

        jar = JarReader(src)
        # Initial copy
        f = DeflatedFile(jar['test'])
        f.copy(dest)

        # Ensure subsequent copies won't trigger writes
        f.copy(DestNoWrite(dest))
        self.assertEqual('test', open(dest, 'rb').read())

        # When using a different file with the same content, no copy should
        # occur
        f = DeflatedFile(jar['test2'])
        f.copy(DestNoWrite(dest))
        self.assertEqual('test', open(dest, 'rb').read())

        # Double check that under conditions where a copy occurs, we would get
        # an exception.
        f = DeflatedFile(jar['fooo'])
        self.assertRaises(RuntimeError, f.copy, DestNoWrite(dest))
Esempio n. 3
0
    def test_deflated_file_open(self):
        '''
        Test whether DeflatedFile.open returns an appropriately reset file
        object.
        '''
        src = self.tmppath('src.jar')
        content = b''.join(samples)
        with JarWriter(src) as jar:
            jar.add('content', content)

        f = DeflatedFile(JarReader(src)['content'])
        self.assertEqual(content[:42], f.open().read(42))
        self.assertEqual(content, f.open().read())
Esempio n. 4
0
    def test_deflated_file_open(self):
        '''
        Test whether DeflatedFile.open returns an appropriately reset file
        object.
        '''
        src = self.tmppath('src.jar')
        content = ''.join(samples)
        with JarWriter(src) as jar:
            jar.add('content', content)

        f = DeflatedFile(JarReader(src)['content'])
        self.assertEqual(content[:42], f.open().read(42))
        self.assertEqual(content, f.open().read())
Esempio n. 5
0
    def test_deflated_file_open(self):
        """
        Test whether DeflatedFile.open returns an appropriately reset file
        object.
        """
        src = self.tmppath("src.jar")
        content = "".join(samples)
        with JarWriter(src) as jar:
            jar.add("content", content)

        f = DeflatedFile(JarReader(src)["content"])
        self.assertEqual(content[:42], f.open().read(42))
        self.assertEqual(content, f.open().read())
Esempio n. 6
0
    def test_deflated_file_open(self):
        """
        Test whether DeflatedFile.open returns an appropriately reset file
        object.
        """
        src = self.tmppath("src.jar")
        content = b"".join(samples)
        with JarWriter(src) as jar:
            jar.add("content", content)

        f = DeflatedFile(JarReader(src)["content"])
        self.assertEqual(content[:42], f.open().read(42))
        self.assertEqual(content, f.open().read())
Esempio n. 7
0
 def _handle_manifest_entry(self, entry, jars):
     jarpath = None
     if isinstance(entry, ManifestEntryWithRelPath) and \
             urlparse(entry.relpath).scheme == 'jar':
         jarpath, entry = self._unjarize(entry, entry.relpath)
     elif isinstance(entry, ManifestResource) and \
             urlparse(entry.target).scheme == 'jar':
         jarpath, entry = self._unjarize(entry, entry.target)
     if jarpath:
         # Don't defer unpacking the jar file. If we already saw
         # it, take (and remove) it from the registry. If we
         # haven't, try to find it now.
         if self.files.contains(jarpath):
             jar = self.files[jarpath]
             self.files.remove(jarpath)
         else:
             jar = [f for p, f in FileFinder.find(self, jarpath)]
             assert len(jar) == 1
             jar = jar[0]
         if not jarpath in jars:
             base = mozpack.path.splitext(jarpath)[0]
             for j in self._open_jar(jarpath, jar):
                 self.files.add(mozpack.path.join(base, j.filename),
                                DeflatedFile(j))
         jars.add(jarpath)
         self.kind = 'jar'
     return entry
Esempio n. 8
0
    def test_deflated_file(self):
        """
        Check that DeflatedFile.copy yields the proper content in the
        destination file in all situations that trigger different code paths
        (see TestFile.test_file)
        """
        src = self.tmppath("src.jar")
        dest = self.tmppath("dest")

        contents = {}
        with JarWriter(src) as jar:
            for content in samples:
                name = "".join(random.choice(string.letters) for i in xrange(8))
                jar.add(name, content, compress=True)
                contents[name] = content

        for j in JarReader(src):
            f = DeflatedFile(j)
            f.copy(dest)
            self.assertEqual(contents[j.filename], open(dest, "rb").read())
Esempio n. 9
0
    def test_deflated_file(self):
        '''
        Check that DeflatedFile.copy yields the proper content in the
        destination file in all situations that trigger different code paths
        (see TestFile.test_file)
        '''
        src = self.tmppath('src.jar')
        dest = self.tmppath('dest')

        contents = {}
        with JarWriter(src) as jar:
            for content in samples:
                name = ''.join(
                    random.choice(string.letters) for i in xrange(8))
                jar.add(name, content, compress=True)
                contents[name] = content

        for j in JarReader(src):
            f = DeflatedFile(j)
            f.copy(dest)
            self.assertEqual(contents[j.filename], open(dest, 'rb').read())
Esempio n. 10
0
 def _fill_with_omnijar(self, base, jar):
     for j in jar:
         path = mozpack.path.join(base, j.filename)
         if is_manifest(j.filename):
             m = self.files[path] if self.files.contains(path) \
                 else ManifestFile(mozpack.path.dirname(path))
             for e in parse_manifest(None, path, j):
                 m.add(e)
             if not self.files.contains(path):
                 self.files.add(path, m)
             continue
         else:
             self.files.add(path, DeflatedFile(j))
Esempio n. 11
0
    def test_deflated_file(self):
        """
        Check that DeflatedFile.copy yields the proper content in the
        destination file in all situations that trigger different code paths
        (see TestFile.test_file)
        """
        src = self.tmppath("src.jar")
        dest = self.tmppath("dest")

        contents = {}
        with JarWriter(src) as jar:
            for content in samples:
                name = "".join(
                    random.choice(
                        "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
                    for i in range(8))
                jar.add(name, content, compress=True)
                contents[name] = content

        for j in JarReader(src):
            f = DeflatedFile(j)
            f.copy(dest)
            self.assertEqual(contents[j.filename], open(dest, "rb").read())
Esempio n. 12
0
    def test_deflated_file_no_write(self):
        '''
        Test various conditions where DeflatedFile.copy is expected not to
        write in the destination file.
        '''
        src = self.tmppath('src.jar')
        dest = self.tmppath('dest')

        with JarWriter(src) as jar:
            jar.add('test', 'test')
            jar.add('test2', 'test')
            jar.add('fooo', 'fooo')

        jar = JarReader(src)
        # Initial copy
        f = DeflatedFile(jar['test'])
        f.copy(dest)

        # Ensure subsequent copies won't trigger writes
        f.copy(DestNoWrite(dest))
        self.assertEqual('test', open(dest, 'rb').read())

        # When using a different file with the same content, no copy should
        # occur
        f = DeflatedFile(jar['test2'])
        f.copy(DestNoWrite(dest))
        self.assertEqual('test', open(dest, 'rb').read())

        # Double check that under conditions where a copy occurs, we would get
        # an exception.
        f = DeflatedFile(jar['fooo'])
        self.assertRaises(RuntimeError, f.copy, DestNoWrite(dest))
Esempio n. 13
0
def package_fennec_apk(inputs=[],
                       omni_ja=None,
                       classes_dex=None,
                       lib_dirs=[],
                       assets_dirs=[],
                       szip_assets_libs_with=None,
                       root_files=[],
                       verbose=False):
    jarrer = Jarrer(optimize=False)

    # First, take input files.  The contents of the later files overwrites the
    # content of earlier files.
    for input in inputs:
        jar = JarReader(input)
        for file in jar:
            path = file.filename
            if jarrer.contains(path):
                jarrer.remove(path)
            jarrer.add(path, DeflatedFile(file), compress=file.compressed)

    def add(path, file, compress=None):
        abspath = os.path.abspath(file.path)
        if verbose:
            print('Packaging %s from %s' % (path, file.path))
        if not os.path.exists(abspath):
            raise ValueError('File %s not found (looked for %s)' % \
                             (file.path, abspath))
        if jarrer.contains(path):
            jarrer.remove(path)
        jarrer.add(path, file, compress=compress)

    for assets_dir in assets_dirs:
        finder = FileFinder(assets_dir, find_executables=False)
        for p, f in finder.find('**'):
            compress = None  # Take default from Jarrer.
            if p.endswith('.so'):
                # Asset libraries are special.
                if szip_assets_libs_with:
                    # We need to szip libraries before packing.  The file
                    # returned by the finder is not yet opened.  When it is
                    # opened, it will "see" the content updated by szip.
                    subprocess.check_output(
                        [szip_assets_libs_with,
                         mozpath.join(finder.base, p)])

                if f.open().read(4) == 'SeZz':
                    # We need to store (rather than deflate) szipped libraries
                    # (even if we don't szip them ourselves).
                    compress = False
            add(mozpath.join('assets', p), f, compress=compress)

    for lib_dir in lib_dirs:
        finder = FileFinder(lib_dir, find_executables=False)
        for p, f in finder.find('**'):
            add(mozpath.join('lib', p), f)

    for root_file in root_files:
        add(os.path.basename(root_file), File(root_file))

    if omni_ja:
        add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False)

    if classes_dex:
        add('classes.dex', File(classes_dex))

    return jarrer
Esempio n. 14
0
def package_fennec_apk(inputs=[],
                       omni_ja=None,
                       classes_dex=None,
                       lib_dirs=[],
                       assets_dirs=[],
                       features_dirs=[],
                       root_files=[],
                       verbose=False):
    jarrer = Jarrer(optimize=False)

    # First, take input files.  The contents of the later files overwrites the
    # content of earlier files.  Multidexing requires special care: we want a
    # coherent set of classesN.dex files, so we only take DEX files from a
    # single input.  This avoids taking, say, classes{1,2,3}.dex from the first
    # input and only classes{1,2}.dex from the second input, leading to
    # (potentially) duplicated symbols at runtime.
    last_input_with_dex_files = None
    for input in inputs:
        jar = JarReader(input)
        for file in jar:
            path = file.filename

            if mozpath.match(path, '/classes*.dex'):
                last_input_with_dex_files = input
                continue

            if jarrer.contains(path):
                jarrer.remove(path)
            jarrer.add(path, DeflatedFile(file), compress=file.compressed)

    # If we have an input with DEX files, take them all here.
    if last_input_with_dex_files:
        jar = JarReader(last_input_with_dex_files)
        for file in jar:
            path = file.filename

            if not mozpath.match(path, '/classes*.dex'):
                continue

            if jarrer.contains(path):
                jarrer.remove(path)
            jarrer.add(path, DeflatedFile(file), compress=file.compressed)

    def add(path, file, compress=None):
        abspath = os.path.abspath(file.path)
        if verbose:
            print('Packaging %s from %s' % (path, file.path))
        if not os.path.exists(abspath):
            raise ValueError('File %s not found (looked for %s)' % \
                             (file.path, abspath))
        if jarrer.contains(path):
            jarrer.remove(path)
        jarrer.add(path, file, compress=compress)

    for features_dir in features_dirs:
        finder = FileFinder(features_dir)
        for p, f in finder.find('**'):
            add(mozpath.join('assets', 'features', p), f, False)

    for assets_dir in assets_dirs:
        finder = FileFinder(assets_dir)
        for p, f in finder.find('**'):
            compress = None  # Take default from Jarrer.
            if p.endswith('.so'):
                # Asset libraries are special.
                if f.open().read(5)[1:] == '7zXZ':
                    print('%s is already compressed' % p)
                    # We need to store (rather than deflate) compressed libraries
                    # (even if we don't compress them ourselves).
                    compress = False
                elif buildconfig.substs.get('XZ'):
                    cmd = [
                        buildconfig.substs.get('XZ'), '-zkf',
                        mozpath.join(finder.base, p)
                    ]

                    # For now, the mozglue XZStream ELF loader can only support xz files
                    # with a single stream that contains a single block. In xz, there is no
                    # explicit option to set the max block count. Instead, we force xz to use
                    # single thread mode, which results in a single block.
                    cmd.extend(['--threads=1'])

                    bcj = None
                    if buildconfig.substs.get('MOZ_THUMB2'):
                        bcj = '--armthumb'
                    elif buildconfig.substs.get('CPU_ARCH') == 'arm':
                        bcj = '--arm'
                    elif buildconfig.substs.get('CPU_ARCH') == 'x86':
                        bcj = '--x86'

                    if bcj:
                        cmd.extend([bcj])
                    # We need to explicitly specify the LZMA filter chain to ensure consistent builds
                    # across platforms. Note that the dict size must be less then 16MiB per the hardcoded
                    # value in mozglue/linker/XZStream.cpp. This is the default LZMA filter chain for for
                    # xz-utils version 5.0. See:
                    # https://github.com/xz-mirror/xz/blob/v5.0.0/src/liblzma/lzma/lzma_encoder_presets.c
                    # https://github.com/xz-mirror/xz/blob/v5.0.0/src/liblzma/api/lzma/container.h#L31
                    cmd.extend([
                        '--lzma2=dict=8MiB,lc=3,lp=0,pb=2,mode=normal,nice=64,mf=bt4,depth=0'
                    ])
                    print('xz-compressing %s with %s' % (p, ' '.join(cmd)))
                    subprocess.check_output(cmd)
                    os.rename(f.path + '.xz', f.path)
                    compress = False

            add(mozpath.join('assets', p), f, compress=compress)

    for lib_dir in lib_dirs:
        finder = FileFinder(lib_dir)
        for p, f in finder.find('**'):
            add(mozpath.join('lib', p), f)

    for root_file in root_files:
        add(os.path.basename(root_file), File(root_file))

    if omni_ja:
        add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False)

    if classes_dex:
        if buildconfig.substs.get('MOZ_BUILD_MOBILE_ANDROID_WITH_GRADLE'):
            raise ValueError("Fennec APKs built --with-gradle "
                             "should never specify classes.dex")

        add('classes.dex', File(classes_dex))

    return jarrer
def package_fennec_apk(inputs=[],
                       omni_ja=None,
                       classes_dex=None,
                       lib_dirs=[],
                       assets_dirs=[],
                       features_dirs=[],
                       root_files=[],
                       verbose=False):
    jarrer = Jarrer(optimize=False)

    # First, take input files.  The contents of the later files overwrites the
    # content of earlier files.
    for input in inputs:
        jar = JarReader(input)
        for file in jar:
            path = file.filename
            if jarrer.contains(path):
                jarrer.remove(path)
            jarrer.add(path, DeflatedFile(file), compress=file.compressed)

    def add(path, file, compress=None):
        abspath = os.path.abspath(file.path)
        if verbose:
            print('Packaging %s from %s' % (path, file.path))
        if not os.path.exists(abspath):
            raise ValueError('File %s not found (looked for %s)' % \
                             (file.path, abspath))
        if jarrer.contains(path):
            jarrer.remove(path)
        jarrer.add(path, file, compress=compress)

    for features_dir in features_dirs:
        finder = FileFinder(features_dir, find_executables=False)
        for p, f in finder.find('**'):
            add(mozpath.join('assets', 'features', p), f, False)

    for assets_dir in assets_dirs:
        finder = FileFinder(assets_dir, find_executables=False)
        for p, f in finder.find('**'):
            compress = None  # Take default from Jarrer.
            if p.endswith('.so'):
                # Asset libraries are special.
                if f.open().read(5)[1:] == '7zXZ':
                    print('%s is already compressed' % p)
                    # We need to store (rather than deflate) compressed libraries
                    # (even if we don't compress them ourselves).
                    compress = False
                elif buildconfig.substs.get('XZ'):
                    cmd = [
                        buildconfig.substs.get('XZ'), '-zkf',
                        mozpath.join(finder.base, p)
                    ]

                    bcj = None
                    if buildconfig.substs.get('MOZ_THUMB2'):
                        bcj = '--armthumb'
                    elif buildconfig.substs.get('CPU_ARCH') == 'arm':
                        bcj = '--arm'
                    elif buildconfig.substs.get('CPU_ARCH') == 'x86':
                        bcj = '--x86'

                    if bcj:
                        cmd.extend([bcj, '--lzma2'])
                    print('xz-compressing %s with %s' % (p, ' '.join(cmd)))
                    subprocess.check_output(cmd)
                    os.rename(f.path + '.xz', f.path)
                    compress = False

            add(mozpath.join('assets', p), f, compress=compress)

    for lib_dir in lib_dirs:
        finder = FileFinder(lib_dir, find_executables=False)
        for p, f in finder.find('**'):
            add(mozpath.join('lib', p), f)

    for root_file in root_files:
        add(os.path.basename(root_file), File(root_file))

    if omni_ja:
        add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False)

    if classes_dex:
        add('classes.dex', File(classes_dex))

    return jarrer
Esempio n. 16
0
    def test_deflated_file_no_write(self):
        """
        Test various conditions where DeflatedFile.copy is expected not to
        write in the destination file.
        """
        src = self.tmppath("src.jar")
        dest = self.tmppath("dest")

        with JarWriter(src) as jar:
            jar.add("test", b"test")
            jar.add("test2", b"test")
            jar.add("fooo", b"fooo")

        jar = JarReader(src)
        # Initial copy
        f = DeflatedFile(jar["test"])
        f.copy(dest)

        # Ensure subsequent copies won't trigger writes
        f.copy(DestNoWrite(dest))
        self.assertEqual(b"test", open(dest, "rb").read())

        # When using a different file with the same content, no copy should
        # occur
        f = DeflatedFile(jar["test2"])
        f.copy(DestNoWrite(dest))
        self.assertEqual(b"test", open(dest, "rb").read())

        # Double check that under conditions where a copy occurs, we would get
        # an exception.
        f = DeflatedFile(jar["fooo"])
        self.assertRaises(RuntimeError, f.copy, DestNoWrite(dest))
Esempio n. 17
0
def package_fennec_apk(inputs=[],
                       omni_ja=None,
                       lib_dirs=[],
                       assets_dirs=[],
                       features_dirs=[],
                       root_files=[],
                       verbose=False):
    jarrer = Jarrer()

    # First, take input files.  The contents of the later files overwrites the
    # content of earlier files.  Multidexing requires special care: we want a
    # coherent set of classesN.dex files, so we only take DEX files from a
    # single input.  This avoids taking, say, classes{1,2,3}.dex from the first
    # input and only classes{1,2}.dex from the second input, leading to
    # (potentially) duplicated symbols at runtime.
    last_input_with_dex_files = None
    for input in inputs:
        jar = JarReader(input)
        for file in jar:
            path = file.filename

            if mozpath.match(path, '/classes*.dex'):
                last_input_with_dex_files = input
                continue

            if jarrer.contains(path):
                jarrer.remove(path)
            jarrer.add(path, DeflatedFile(file), compress=file.compressed)

    # If we have an input with DEX files, take them all here.
    if last_input_with_dex_files:
        jar = JarReader(last_input_with_dex_files)
        for file in jar:
            path = file.filename

            if not mozpath.match(path, '/classes*.dex'):
                continue

            if jarrer.contains(path):
                jarrer.remove(path)
            jarrer.add(path, DeflatedFile(file), compress=file.compressed)

    def add(path, file, compress=None):
        abspath = os.path.abspath(file.path)
        if verbose:
            print('Packaging %s from %s' % (path, file.path))
        if not os.path.exists(abspath):
            raise ValueError('File %s not found (looked for %s)' %
                             (file.path, abspath))
        if jarrer.contains(path):
            jarrer.remove(path)
        jarrer.add(path, file, compress=compress)

    for features_dir in features_dirs:
        finder = FileFinder(features_dir)
        for p, f in finder.find('**'):
            add(mozpath.join('assets', 'features', p), f, False)

    for assets_dir in assets_dirs:
        finder = FileFinder(assets_dir)
        for p, f in finder.find('**'):
            add(mozpath.join('assets', p), f)

    for lib_dir in lib_dirs:
        finder = FileFinder(lib_dir)
        for p, f in finder.find('**'):
            add(mozpath.join('lib', p), f)

    for root_file in root_files:
        add(os.path.basename(root_file), File(root_file))

    if omni_ja:
        add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False)

    return jarrer