Exemple #1
0
    def write(self, pack_sha, write):
        """Write the index file using the given write method
        :param pack_sha: binary sha over the whole pack that we index
        :return: sha1 binary sha over all index file contents"""
        # sort for sha1 hash
        self._objs.sort(key=lambda o: o[0])

        sha_writer = FlexibleSha1Writer(write)
        sha_write = sha_writer.write
        sha_write(PackIndexFile.index_v2_signature)
        sha_write(pack(">L", PackIndexFile.index_version_default))

        # fanout
        tmplist = list((0, ) * 256)  # fanout or list with 64 bit offsets
        for t in self._objs:
            tmplist[byte_ord(t[0][0])] += 1
        # END prepare fanout
        for i in xrange(255):
            v = tmplist[i]
            sha_write(pack('>L', v))
            tmplist[i + 1] += v
        # END write each fanout entry
        sha_write(pack('>L', tmplist[255]))

        # sha1 ordered
        # save calls, that is push them into c
        sha_write(b''.join(t[0] for t in self._objs))

        # crc32
        for t in self._objs:
            sha_write(pack('>L', t[1] & 0xffffffff))
        # END for each crc

        tmplist = list()
        # offset 32
        for t in self._objs:
            ofs = t[2]
            if ofs > 0x7fffffff:
                tmplist.append(ofs)
                ofs = 0x80000000 + len(tmplist) - 1
            # END hande 64 bit offsets
            sha_write(pack('>L', ofs & 0xffffffff))
        # END for each offset

        # offset 64
        for ofs in tmplist:
            sha_write(pack(">Q", ofs))
        # END for each offset

        # trailer
        assert (len(pack_sha) == 20)
        sha_write(pack_sha)
        sha = sha_writer.sha(as_hex=False)
        write(sha)
        return sha
Exemple #2
0
    def write_pack(cls,
                   object_iter,
                   pack_write,
                   index_write=None,
                   object_count=None,
                   zlib_compression=zlib.Z_BEST_SPEED):
        """
        Create a new pack by putting all objects obtained by the object_iterator
        into a pack which is written using the pack_write method.
        The respective index is produced as well if index_write is not Non.

        :param object_iter: iterator yielding odb output objects
        :param pack_write: function to receive strings to write into the pack stream
        :param indx_write: if not None, the function writes the index file corresponding
            to the pack.
        :param object_count: if you can provide the amount of objects in your iteration,
            this would be the place to put it. Otherwise we have to pre-iterate and store
            all items into a list to get the number, which uses more memory than necessary.
        :param zlib_compression: the zlib compression level to use
        :return: tuple(pack_sha, index_binsha) binary sha over all the contents of the pack
            and over all contents of the index. If index_write was None, index_binsha will be None

        **Note:** The destination of the write functions is up to the user. It could
        be a socket, or a file for instance

        **Note:** writes only undeltified objects"""
        objs = object_iter
        if not object_count:
            if not isinstance(object_iter, (tuple, list)):
                objs = list(object_iter)
            # END handle list type
            object_count = len(objs)
        # END handle object

        pack_writer = FlexibleSha1Writer(pack_write)
        pwrite = pack_writer.write
        ofs = 0  # current offset into the pack file
        index = None
        wants_index = index_write is not None

        # write header
        pwrite(
            pack('>LLL', PackFile.pack_signature,
                 PackFile.pack_version_default, object_count))
        ofs += 12

        if wants_index:
            index = IndexWriter()
        # END handle index header

        actual_count = 0
        for obj in objs:
            actual_count += 1
            crc = 0

            # object header
            hdr = create_pack_object_header(obj.type_id, obj.size)
            if index_write:
                crc = crc32(hdr)
            else:
                crc = None
            # END handle crc
            pwrite(hdr)

            # data stream
            zstream = zlib.compressobj(zlib_compression)
            ostream = obj.stream
            br, bw, crc = write_stream_to_pack(ostream.read,
                                               pwrite,
                                               zstream,
                                               base_crc=crc)
            assert (br == obj.size)
            if wants_index:
                index.append(obj.binsha, crc, ofs)
            # END handle index

            ofs += len(hdr) + bw
            if actual_count == object_count:
                break
            # END abort once we are done
        # END for each object

        if actual_count != object_count:
            raise ValueError(
                "Expected to write %i objects into pack, but received only %i from iterators"
                % (object_count, actual_count))
        # END count assertion

        # write footer
        pack_sha = pack_writer.sha(as_hex=False)
        assert len(pack_sha) == 20
        pack_write(pack_sha)
        ofs += len(pack_sha)  # just for completeness ;)

        index_sha = None
        if wants_index:
            index_sha = index.write(pack_sha, index_write)
        # END handle index

        return pack_sha, index_sha