def write_pack(cls, object_iter, pack_write, index_write=None,
object_count = None, zlib_compression = zlib.Z_BEST_SPEED):
"""
Create a new pack by putting all objects obtained by the object_iterator
into a pack which is written using the pack_write method.
The respective index is produced as well if index_write is not Non.
:param object_iter: iterator yielding odb output objects
:param pack_write: function to receive strings to write into the pack stream
:param indx_write: if not None, the function writes the index file corresponding
to the pack.
:param object_count: if you can provide the amount of objects in your iteration,
this would be the place to put it. Otherwise we have to pre-iterate and store
all items into a list to get the number, which uses more memory than necessary.
:param zlib_compression: the zlib compression level to use
:return: tuple(pack_sha, index_binsha) binary sha over all the contents of the pack
and over all contents of the index. If index_write was None, index_binsha will be None
:note: The destination of the write functions is up to the user. It could
be a socket, or a file for instance
:note: writes only undeltified objects"""
objs = object_iter
if not object_count:
if not isinstance(object_iter, (tuple, list)):
objs = list(object_iter)
#END handle list type
object_count = len(objs)
#END handle object
pack_writer = FlexibleSha1Writer(pack_write)
pwrite = pack_writer.write
ofs = 0 # current offset into the pack file
index = None
wants_index = index_write is not None
# write header
pwrite(pack('>LLL', PackFile.pack_signature, PackFile.pack_version_default, object_count))
ofs += 12
if wants_index:
index = IndexWriter()
#END handle index header
actual_count = 0
for obj in objs:
actual_count += 1
crc = 0
# object header
hdr = create_pack_object_header(obj.type_id, obj.size)
if index_write:
crc = crc32(hdr)
else:
crc = None
#END handle crc
pwrite(hdr)
# data stream
zstream = zlib.compressobj(zlib_compression)
ostream = obj.stream
br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, base_crc = crc)
assert(br == obj.size)
if wants_index:
index.append(obj.binsha, crc, ofs)
#END handle index
ofs += len(hdr) + bw
if actual_count == object_count:
break
#END abort once we are done
#END for each object
if actual_count != object_count:
raise ValueError("Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count))
#END count assertion
# write footer
pack_sha = pack_writer.sha(as_hex = False)
assert len(pack_sha) == 20
pack_write(pack_sha)
ofs += len(pack_sha) # just for completeness ;)
index_sha = None
if wants_index:
index_sha = index.write(pack_sha, index_write)
#END handle index
return pack_sha, index_sha
def write_pack(cls,
object_iter,
pack_write,
index_write=None,
object_count=None,
zlib_compression=zlib.Z_BEST_SPEED):
"""
Create a new pack by putting all objects obtained by the object_iterator
into a pack which is written using the pack_write method.
The respective index is produced as well if index_write is not Non.
:param object_iter: iterator yielding odb output objects
:param pack_write: function to receive strings to write into the pack stream
:param indx_write: if not None, the function writes the index file corresponding
to the pack.
:param object_count: if you can provide the amount of objects in your iteration,
this would be the place to put it. Otherwise we have to pre-iterate and store
all items into a list to get the number, which uses more memory than necessary.
:param zlib_compression: the zlib compression level to use
:return: tuple(pack_sha, index_binsha) binary sha over all the contents of the pack
and over all contents of the index. If index_write was None, index_binsha will be None
:note: The destination of the write functions is up to the user. It could
be a socket, or a file for instance
:note: writes only undeltified objects"""
objs = object_iter
if not object_count:
if not isinstance(object_iter, (tuple, list)):
objs = list(object_iter)
#END handle list type
object_count = len(objs)
#END handle object
pack_writer = FlexibleSha1Writer(pack_write)
pwrite = pack_writer.write
ofs = 0 # current offset into the pack file
index = None
wants_index = index_write is not None
# write header
pwrite(
pack('>LLL', PackFile.pack_signature,
PackFile.pack_version_default, object_count))
ofs += 12
if wants_index:
index = IndexWriter()
#END handle index header
actual_count = 0
for obj in objs:
actual_count += 1
crc = 0
# object header
hdr = create_pack_object_header(obj.type_id, obj.size)
if index_write:
crc = crc32(hdr)
else:
crc = None
#END handle crc
pwrite(hdr)
# data stream
zstream = zlib.compressobj(zlib_compression)
ostream = obj.stream
br, bw, crc = write_stream_to_pack(ostream.read,
pwrite,
zstream,
base_crc=crc)
assert (br == obj.size)
if wants_index:
index.append(obj.binsha, crc, ofs)
#END handle index
ofs += len(hdr) + bw
if actual_count == object_count:
break
#END abort once we are done
#END for each object
if actual_count != object_count:
raise ValueError(
"Expected to write %i objects into pack, but received only %i from iterators"
% (object_count, actual_count))
#END count assertion
# write footer
pack_sha = pack_writer.sha(as_hex=False)
assert len(pack_sha) == 20
pack_write(pack_sha)
ofs += len(pack_sha) # just for completeness ;)
index_sha = None
if wants_index:
index_sha = index.write(pack_sha, index_write)
#END handle index
return pack_sha, index_sha
def __init__(self, fd):
super(FDCompressedSha1Writer, self).__init__()
self.fd = fd
self.zip = zlib.compressobj(zlib.Z_BEST_SPEED)
def __init__(self):
Sha1Writer.__init__(self)
self.buf = StringIO()
self.zip = zlib.compressobj(zlib.Z_BEST_SPEED)
def __init__(self, fd):
super(FDCompressedSha1Writer, self).__init__()
self.fd = fd
self.zip = zlib.compressobj(zlib.Z_BEST_SPEED)
def __init__(self):
Sha1Writer.__init__(self)
self.buf = StringIO()
self.zip = zlib.compressobj(zlib.Z_BEST_SPEED)
