def move_in_thin_pack(self, path):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
data = ThinPackData(self.get_raw, path)
# Write index for the thin pack (do we really need this?)
temppath = os.path.join(
self.pack_dir,
sha_to_hex(urllib2.randombytes(20)) + ".tempidx")
data.create_index_v2(temppath)
p = Pack.from_objects(data, load_pack_index(temppath))
try:
# Write a full pack version
temppath = os.path.join(
self.pack_dir,
sha_to_hex(urllib2.randombytes(20)) + ".temppack")
write_pack(temppath, ((o, None) for o in p.iterobjects()), len(p))
finally:
p.close()
pack_sha = load_pack_index(temppath + ".idx").objects_sha1()
newbasename = os.path.join(self.pack_dir, "pack-%s" % pack_sha)
os.rename(temppath + ".pack", newbasename + ".pack")
os.rename(temppath + ".idx", newbasename + ".idx")
final_pack = Pack(newbasename)
self._add_known_pack(final_pack)
return final_pack
def move_in_thin_pack(self, path):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
data = ThinPackData(self.get_raw, path)
# Write index for the thin pack (do we really need this?)
temppath = os.path.join(self.pack_dir,
sha_to_hex(urllib2.randombytes(20))+".tempidx")
data.create_index_v2(temppath)
p = Pack.from_objects(data, load_pack_index(temppath))
try:
# Write a full pack version
temppath = os.path.join(self.pack_dir,
sha_to_hex(urllib2.randombytes(20))+".temppack")
write_pack(temppath, ((o, None) for o in p.iterobjects()), len(p))
finally:
p.close()
pack_sha = load_pack_index(temppath+".idx").objects_sha1()
newbasename = os.path.join(self.pack_dir, "pack-%s" % pack_sha)
os.rename(temppath+".pack", newbasename+".pack")
os.rename(temppath+".idx", newbasename+".idx")
final_pack = Pack(newbasename)
self._add_known_pack(final_pack)
return final_pack
def data(self):
"""The pack data object being used."""
if self._data is None:
self._data = self._data_load()
self._data.pack = self
assert len(self.index) == len(self._data)
idx_stored_checksum = self.index.get_pack_checksum()
data_stored_checksum = self._data.get_stored_checksum()
if idx_stored_checksum != data_stored_checksum:
raise ChecksumMismatch(sha_to_hex(idx_stored_checksum),
sha_to_hex(data_stored_checksum))
return self._data
def get_raw(self, name):
"""Obtain the raw text for an object.
:param name: sha for the object.
:return: tuple with numeric type and object contents.
"""
if len(name) == 40:
sha = hex_to_sha(name)
hexsha = name
elif len(name) == 20:
sha = name
hexsha = None
else:
raise AssertionError("Invalid object name %r" % name)
for pack in self.packs:
try:
return pack.get_raw(sha)
except KeyError:
pass
if hexsha is None:
hexsha = sha_to_hex(name)
ret = self._get_loose_object(hexsha)
if ret is not None:
return ret.type_num, ret.as_raw_string()
raise KeyError(hexsha)
def read_objects(self):
"""Read the objects in this pack file.
:raise AssertionError: if there is an error in the pack format.
:raise ChecksumMismatch: if the checksum of the pack contents does not
match the checksum in the pack trailer.
:raise zlib.error: if an error occurred during zlib decompression.
:raise IOError: if an error occurred writing to the output file.
"""
pack_version, self._num_objects = read_pack_header(self.read)
for i in xrange(self._num_objects):
type, uncomp, comp_len, unused = unpack_object(self.read, self.recv)
yield type, uncomp, comp_len
# prepend any unused data to current read buffer
buf = StringIO()
buf.write(unused)
buf.write(self._rbuf.read())
buf.seek(0)
self._rbuf = buf
pack_sha = sha_to_hex(''.join([c for c in self._trailer]))
calculated_sha = self.sha.hexdigest()
if pack_sha != calculated_sha:
raise ChecksumMismatch(pack_sha, calculated_sha)
def get_raw(self, name):
"""Obtain the raw text for an object.
:param name: sha for the object.
:return: tuple with numeric type and object contents.
"""
if len(name) == 40:
sha = hex_to_sha(name)
hexsha = name
elif len(name) == 20:
sha = name
hexsha = None
else:
raise AssertionError("Invalid object name %r" % name)
for pack in self.packs:
try:
return pack.get_raw(sha)
except KeyError:
pass
if hexsha is None:
hexsha = sha_to_hex(name)
ret = self._get_loose_object(hexsha)
if ret is not None:
return ret.type_num, ret.as_raw_string()
raise KeyError(hexsha)
def read_cache_entry(f):
"""Read an entry from a cache file.
:param f: File-like object to read from
:return: tuple with: device, inode, mode, uid, gid, size, sha, flags
"""
beginoffset = f.tell()
ctime = read_cache_time(f)
mtime = read_cache_time(f)
(dev, ino, mode, uid, gid, size, sha, flags, ) = \
struct.unpack(">LLLLLL20sH", f.read(20 + 4 * 6 + 2))
name = f.read((flags & 0x0fff))
# Padding:
real_size = ((f.tell() - beginoffset + 8) & ~7)
data = f.read((beginoffset + real_size) - f.tell())
return (name, ctime, mtime, dev, ino, mode, uid, gid, size,
sha_to_hex(sha), flags & ~0x0fff)
class ThinPackData(PackData):
"""PackData for thin packs, which require an ObjectStore for resolving."""
def __init__(self, resolve_ext_ref, *args, **kwargs):
super(ThinPackData, self).__init__(*args, **kwargs)
self.resolve_ext_ref = resolve_ext_ref
@classmethod
def from_file(cls, resolve_ext_ref, file, size):
return cls(resolve_ext_ref, str(file), file=file, size=size)
def get_ref(self, sha):
"""Resolve a reference looking in both this pack and the store."""
try:
# As part of completing a pack we create a Pack object with a
# ThinPackData and a full PackIndex, so check in the index first if
# possible.
# TODO(dborowitz): reevaluate this when the pack completion code is
# rewritten.
return super(ThinPackData, self).get_ref(sha)
except KeyError:
type, obj = self.resolve_ext_ref(sha)
return None, type, obj
def iterentries(self, progress=None):
"""Yield entries summarizing the contents of this pack.
:param progress: Progress function, called with current and
total object count.
This will yield tuples with (sha, offset, crc32)
"""
found = {}
postponed = defaultdict(list)
class Postpone(Exception):
"""Raised to postpone delta resolving."""
def __init__(self, sha):
self.sha = sha
def get_ref_text(sha):
assert len(sha) == 20
if sha in found:
offset = found[sha]
type, obj = self.get_object_at(offset)
return offset, type, obj
try:
return self.get_ref(sha)
except KeyError:
raise Postpone(sha)
extra = []
todo = chain(self.iterobjects(progress=progress), extra)
for (offset, type, obj, crc32) in todo:
assert isinstance(offset, int)
if obj is None:
# Inflate postponed delta
obj, type = self.get_object_at(offset)
assert isinstance(type, int)
assert isinstance(obj, list) or isinstance(obj, tuple)
try:
type, obj = self.resolve_object(offset, type, obj, get_ref_text)
except Postpone, e:
# Save memory by not storing the inflated obj in postponed
postponed[e.sha].append((offset, type, None, crc32))
else:
sha = obj_sha(type, obj)
found[sha] = offset
yield sha, offset, crc32
extra.extend(postponed.pop(sha, []))
if postponed:
raise KeyError([sha_to_hex(h) for h in postponed.keys()])
def check_sha(self):
stored = self.f.read(20)
if stored != self.sha1.digest():
raise ChecksumMismatch(self.sha1.hexdigest(), sha_to_hex(stored))
