def write(self, pack_sha, write):
"""Write the index file using the given write method
:param pack_sha: binary sha over the whole pack that we index
:return: sha1 binary sha over all index file contents"""
# sort for sha1 hash
self._objs.sort(key=lambda o: o[0])
sha_writer = FlexibleSha1Writer(write)
sha_write = sha_writer.write
sha_write(PackIndexFile.index_v2_signature)
sha_write(pack(">L", PackIndexFile.index_version_default))
# fanout
tmplist = list((0, ) * 256) # fanout or list with 64 bit offsets
for t in self._objs:
tmplist[byte_ord(t[0][0])] += 1
# END prepare fanout
for i in xrange(255):
v = tmplist[i]
sha_write(pack('>L', v))
tmplist[i + 1] += v
# END write each fanout entry
sha_write(pack('>L', tmplist[255]))
# sha1 ordered
# save calls, that is push them into c
sha_write(b''.join(t[0] for t in self._objs))
# crc32
for t in self._objs:
sha_write(pack('>L', t[1] & 0xffffffff))
# END for each crc
tmplist = list()
# offset 32
for t in self._objs:
ofs = t[2]
if ofs > 0x7fffffff:
tmplist.append(ofs)
ofs = 0x80000000 + len(tmplist) - 1
# END hande 64 bit offsets
sha_write(pack('>L', ofs & 0xffffffff))
# END for each offset
# offset 64
for ofs in tmplist:
sha_write(pack(">Q", ofs))
# END for each offset
# trailer
assert (len(pack_sha) == 20)
sha_write(pack_sha)
sha = sha_writer.sha(as_hex=False)
write(sha)
return sha
def write_pack(cls,
object_iter,
pack_write,
index_write=None,
object_count=None,
zlib_compression=zlib.Z_BEST_SPEED):
"""
Create a new pack by putting all objects obtained by the object_iterator
into a pack which is written using the pack_write method.
The respective index is produced as well if index_write is not Non.
:param object_iter: iterator yielding odb output objects
:param pack_write: function to receive strings to write into the pack stream
:param indx_write: if not None, the function writes the index file corresponding
to the pack.
:param object_count: if you can provide the amount of objects in your iteration,
this would be the place to put it. Otherwise we have to pre-iterate and store
all items into a list to get the number, which uses more memory than necessary.
:param zlib_compression: the zlib compression level to use
:return: tuple(pack_sha, index_binsha) binary sha over all the contents of the pack
and over all contents of the index. If index_write was None, index_binsha will be None
**Note:** The destination of the write functions is up to the user. It could
be a socket, or a file for instance
**Note:** writes only undeltified objects"""
objs = object_iter
if not object_count:
if not isinstance(object_iter, (tuple, list)):
objs = list(object_iter)
# END handle list type
object_count = len(objs)
# END handle object
pack_writer = FlexibleSha1Writer(pack_write)
pwrite = pack_writer.write
ofs = 0 # current offset into the pack file
index = None
wants_index = index_write is not None
# write header
pwrite(
pack('>LLL', PackFile.pack_signature,
PackFile.pack_version_default, object_count))
ofs += 12
if wants_index:
index = IndexWriter()
# END handle index header
actual_count = 0
for obj in objs:
actual_count += 1
crc = 0
# object header
hdr = create_pack_object_header(obj.type_id, obj.size)
if index_write:
crc = crc32(hdr)
else:
crc = None
# END handle crc
pwrite(hdr)
# data stream
zstream = zlib.compressobj(zlib_compression)
ostream = obj.stream
br, bw, crc = write_stream_to_pack(ostream.read,
pwrite,
zstream,
base_crc=crc)
assert (br == obj.size)
if wants_index:
index.append(obj.binsha, crc, ofs)
# END handle index
ofs += len(hdr) + bw
if actual_count == object_count:
break
# END abort once we are done
# END for each object
if actual_count != object_count:
raise ValueError(
"Expected to write %i objects into pack, but received only %i from iterators"
% (object_count, actual_count))
# END count assertion
# write footer
pack_sha = pack_writer.sha(as_hex=False)
assert len(pack_sha) == 20
pack_write(pack_sha)
ofs += len(pack_sha) # just for completeness ;)
index_sha = None
if wants_index:
index_sha = index.write(pack_sha, index_write)
# END handle index
return pack_sha, index_sha