def test_thin_from_file(self):
test_sha = "1" * 40
def resolve(sha):
self.assertEqual(test_sha, sha)
return 3, "data"
path = os.path.join(self.datadir, "pack-%s.pack" % pack1_sha)
data = ThinPackData.from_file(resolve, open(path), os.path.getsize(path))
idx = self.get_pack_index(pack1_sha)
Pack.from_objects(data, idx)
self.assertEqual((None, 3, "data"), data.get_ref(test_sha))
def test_checksum_mismatch(self):
data = self.get_pack_data(pack1_sha)
index = self.get_pack_index(pack1_sha)
Pack.from_objects(data, index).check_length_and_checksum()
data._file.seek(0)
bad_file = BytesIO(data._file.read()[:-20] + ('\xff' * 20))
bad_data = PackData('', file=bad_file)
bad_pack = Pack.from_lazy_objects(lambda: bad_data, lambda: index)
self.assertRaises(ChecksumMismatch, lambda: bad_pack.data)
self.assertRaises(ChecksumMismatch, lambda:
bad_pack.check_length_and_checksum())
def test_checksum_mismatch(self):
with self.get_pack_data(pack1_sha) as data:
index = self.get_pack_index(pack1_sha)
Pack.from_objects(data, index).check_length_and_checksum()
data._file.seek(0)
bad_file = BytesIO(data._file.read()[:-20] + ('\xff' * 20))
bad_data = PackData('', file=bad_file)
bad_pack = Pack.from_lazy_objects(lambda: bad_data, lambda: index)
self.assertRaises(ChecksumMismatch, lambda: bad_pack.data)
self.assertRaises(ChecksumMismatch, lambda:
bad_pack.check_length_and_checksum())
def test_length_mismatch(self):
data = self.get_pack_data(pack1_sha)
index = self.get_pack_index(pack1_sha)
Pack.from_objects(data, index).check_length_and_checksum()
data._file.seek(12)
bad_file = BytesIO()
write_pack_header(bad_file, 9999)
bad_file.write(data._file.read())
bad_file = BytesIO(bad_file.getvalue())
bad_data = PackData('', file=bad_file)
bad_pack = Pack.from_lazy_objects(lambda: bad_data, lambda: index)
self.assertRaises(AssertionError, lambda: bad_pack.data)
self.assertRaises(AssertionError,
lambda: bad_pack.check_length_and_checksum())
def test_length_mismatch(self):
with self.get_pack_data(pack1_sha) as data:
index = self.get_pack_index(pack1_sha)
Pack.from_objects(data, index).check_length_and_checksum()
data._file.seek(12)
bad_file = BytesIO()
write_pack_header(bad_file, 9999)
bad_file.write(data._file.read())
bad_file = BytesIO(bad_file.getvalue())
bad_data = PackData('', file=bad_file)
bad_pack = Pack.from_lazy_objects(lambda: bad_data, lambda: index)
self.assertRaises(AssertionError, lambda: bad_pack.data)
self.assertRaises(AssertionError,
lambda: bad_pack.check_length_and_checksum())
def commit():
if pf.tell() == 0:
pf.close()
return None
pf.seek(0)
p = PackData(pf.name, pf)
entries = p.sorted_entries()
basename = iter_sha1(entry[0] for entry in entries).decode('ascii')
idxf = tempfile.SpooledTemporaryFile()
checksum = p.get_stored_checksum()
write_pack_index_v2(idxf, entries, checksum)
idxf.seek(0)
idx = load_pack_index_file(basename + '.idx', idxf)
for pack in self.packs:
if pack.get_stored_checksum() == p.get_stored_checksum():
p.close()
idx.close()
return pack
pf.seek(0)
idxf.seek(0)
self._upload_pack(basename, pf, idxf)
final_pack = Pack.from_objects(p, idx)
self._add_cached_pack(basename, final_pack)
return final_pack
def _update_pack_cache(self):
pack_files = set(self._pack_names())
new_packs = []
for basename in pack_files:
pack_name = basename + ".pack"
if basename not in self._pack_cache:
try:
size = self.pack_transport.stat(pack_name).st_size
except TransportNotPossible:
f = self.pack_transport.get(pack_name)
# TODO(jelmer): Don't read entire file into memory?
f = BytesIO(f.read())
pd = PackData(pack_name, f)
else:
pd = PackData(
pack_name, self.pack_transport.get(pack_name),
size=size)
idxname = basename + ".idx"
idx = load_pack_index_file(
idxname, self.pack_transport.get(idxname))
pack = Pack.from_objects(pd, idx)
pack._basename = basename
self._pack_cache[basename] = pack
new_packs.append(pack)
# Remove disappeared pack files
for f in set(self._pack_cache) - pack_files:
self._pack_cache.pop(f).close()
return new_packs
def move_in_pack(self, f):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
f.seek(0)
p = PackData("", f, len(f.getvalue()))
entries = p.sorted_entries()
basename = "pack-%s" % iter_sha1(entry[0]
for entry in entries).decode('ascii')
p._filename = basename + ".pack"
f.seek(0)
self.pack_transport.put_file(basename + ".pack", f)
idxfile = self.pack_transport.open_write_stream(basename + ".idx")
try:
write_pack_index_v2(idxfile, entries, p.get_stored_checksum())
finally:
idxfile.close()
idxfile = self.pack_transport.get(basename + ".idx")
idx = load_pack_index_file(basename + ".idx", idxfile)
final_pack = Pack.from_objects(p, idx)
final_pack._basename = basename
self._add_known_pack(basename, final_pack)
return final_pack
def move_in_thin_pack(self, path):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
data = PackData(path)
# Write index for the thin pack (do we really need this?)
temppath = os.path.join(self.pack_dir,
sha_to_hex(urllib2.randombytes(20))+".tempidx")
data.create_index_v2(temppath, self.get_raw)
p = Pack.from_objects(data, load_pack_index(temppath))
# Write a full pack version
temppath = os.path.join(self.pack_dir,
sha_to_hex(urllib2.randombytes(20))+".temppack")
write_pack(temppath, ((o, None) for o in p.iterobjects(self.get_raw)),
len(p))
pack_sha = load_pack_index(temppath+".idx").objects_sha1()
newbasename = os.path.join(self.pack_dir, "pack-%s" % pack_sha)
os.rename(temppath+".pack", newbasename+".pack")
os.rename(temppath+".idx", newbasename+".idx")
self._add_known_pack(newbasename)
def move_in_thin_pack(self, path):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
data = PackData(path)
# Write index for the thin pack (do we really need this?)
temppath = os.path.join(self.pack_dir,
sha_to_hex(urllib2.randombytes(20))+".tempidx")
data.create_index_v2(temppath, self.get_raw)
p = Pack.from_objects(data, load_pack_index(temppath))
# Write a full pack version
temppath = os.path.join(self.pack_dir,
sha_to_hex(urllib2.randombytes(20))+".temppack")
write_pack(temppath, ((o, None) for o in p.iterobjects(self.get_raw)),
len(p))
pack_sha = load_pack_index(temppath+".idx").objects_sha1()
newbasename = os.path.join(self.pack_dir, "pack-%s" % pack_sha)
os.rename(temppath+".pack", newbasename+".pack")
os.rename(temppath+".idx", newbasename+".idx")
self._add_known_pack(newbasename)
def _update_pack_cache(self):
pack_files = set()
pack_dir_contents = self._pack_names()
for name in pack_dir_contents:
if name.startswith("pack-") and name.endswith(".pack"):
# verify that idx exists first (otherwise the pack was not yet
# fully written)
idx_name = os.path.splitext(name)[0] + ".idx"
if idx_name in pack_dir_contents:
pack_files.add(os.path.splitext(name)[0])
new_packs = []
for basename in pack_files:
pack_name = basename + ".pack"
if basename not in self._pack_cache:
try:
size = self.pack_transport.stat(pack_name).st_size
except TransportNotPossible:
f = self.pack_transport.get(pack_name)
pd = PackData(pack_name, f)
else:
pd = PackData(pack_name,
self.pack_transport.get(pack_name),
size=size)
idxname = basename + ".idx"
idx = load_pack_index_file(idxname,
self.pack_transport.get(idxname))
pack = Pack.from_objects(pd, idx)
pack._basename = basename
self._pack_cache[basename] = pack
new_packs.append(pack)
# Remove disappeared pack files
for f in set(self._pack_cache) - pack_files:
self._pack_cache.pop(f).close()
return new_packs
def test_checksum_mismatch(self):
with self.get_pack_data(pack1_sha) as data:
index = self.get_pack_index(pack1_sha)
with Pack.from_objects(data, index) as p:
p.check_length_and_checksum()
data._file.seek(0)
with BytesIO(data._file.read()[:-20] + (b'\xff' * 20)) as bad_file:
with PackData('', file=bad_file) as bad_data:
with Pack.from_lazy_objects(lambda: bad_data, lambda: index) as bad_pack:
self.assertRaises(ChecksumMismatch, lambda: bad_pack.data)
self.assertRaises(ChecksumMismatch, lambda:
bad_pack.check_length_and_checksum())
def test_length_mismatch(self):
with self.get_pack_data(pack1_sha) as data:
index = self.get_pack_index(pack1_sha)
with Pack.from_objects(data, index) as p:
p.check_length_and_checksum()
data._file.seek(12)
with BytesIO() as bad_file:
write_pack_header(bad_file, 9999)
bad_file.write(data._file.read())
with BytesIO(bad_file.getvalue()) as badder_file:
with PackData('', file=badder_file) as bad_data:
with Pack.from_lazy_objects(lambda: bad_data, lambda: index) as bad_pack:
self.assertRaises(AssertionError, lambda: bad_pack.data)
self.assertRaises(AssertionError,
lambda: bad_pack.check_length_and_checksum())
def pack_info_create(pack_data, pack_index):
pack = Pack.from_objects(pack_data, pack_index)
info = {}
for obj in pack.iterobjects():
# Commit
if obj.type_num == Commit.type_num:
info[obj.id] = (obj.type_num, obj.parents, obj.tree)
# Tree
elif obj.type_num == Tree.type_num:
shas = [(s, n, not stat.S_ISDIR(m)) for
n, m, s in obj.iteritems() if not S_ISGITLINK(m)]
info[obj.id] = (obj.type_num, shas)
# Blob
elif obj.type_num == Blob.type_num:
info[obj.id] = None
# Tag
elif obj.type_num == Tag.type_num:
info[obj.id] = (obj.type_num, obj.object[1])
return zlib.compress(json_dumps(info))
def _load_packs(self):
ret = []
for name in self._pack_names():
if name.startswith("pack-") and name.endswith(".pack"):
try:
size = self.pack_transport.stat(name).st_size
except TransportNotPossible:
f = self.pack_transport.get(name)
pd = PackData(name, f)
else:
pd = PackData(name,
self.pack_transport.get(name),
size=size)
idxname = name.replace(".pack", ".idx")
idx = load_pack_index_file(idxname,
self.pack_transport.get(idxname))
pack = Pack.from_objects(pd, idx)
pack._basename = idxname[:-4]
ret.append(pack)
return ret