def test_copy(self):
origpack = self.get_pack(pack1_sha)
try:
self.assertSucceeds(origpack.index.check)
basename = os.path.join(self.tempdir, 'Elch')
write_pack(basename, [(x, '') for x in origpack.iterobjects()],
len(origpack))
newpack = Pack(basename)
try:
self.assertEquals(origpack, newpack)
self.assertSucceeds(newpack.index.check)
self.assertEquals(origpack.name(), newpack.name())
self.assertEquals(origpack.index.get_pack_checksum(),
newpack.index.get_pack_checksum())
wrong_version = origpack.index.version != newpack.index.version
orig_checksum = origpack.index.get_stored_checksum()
new_checksum = newpack.index.get_stored_checksum()
self.assertTrue(wrong_version or orig_checksum == new_checksum)
finally:
newpack.close()
finally:
origpack.close()
def _complete_thin_pack(self, f, path, copier, indexer):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param f: Open file object for the pack.
:param path: Path to the pack file.
:param copier: A PackStreamCopier to use for writing pack data.
:param indexer: A PackIndexer for indexing the pack.
"""
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.close()
# Move the pack in.
entries.sort()
pack_base_name = os.path.join(
self.pack_dir, 'pack-' + iter_sha1(e[0] for e in entries))
os.rename(path, pack_base_name + '.pack')
# Write the index.
index_file = GitFile(pack_base_name + '.idx', 'wb')
try:
write_pack_index_v2(index_file, entries, pack_sha)
index_file.close()
finally:
index_file.abort()
# Add the pack to the store and return it.
final_pack = Pack(pack_base_name)
final_pack.check_length_and_checksum()
self._add_known_pack(final_pack)
return final_pack
def fetch_objects(self, determine_wants, graph_walker, progress=None):
fd, path = tempfile.mkstemp(suffix=".pack")
self.fetch_pack(determine_wants, graph_walker, lambda x: os.write(fd, x), progress)
os.close(fd)
basename = path[:-len(".pack")]
p = PackData(path)
p.create_index_v2(basename+".idx")
pack = Pack(basename)
os.remove(path)
return (len(p), pack.iterobjects())
def test_checksum_mismatch(self):
data = self.get_pack_data(pack1_sha)
index = self.get_pack_index(pack1_sha)
Pack.from_objects(data, index).check_length_and_checksum()
data._file.seek(0)
bad_file = BytesIO(data._file.read()[:-20] + ('\xff' * 20))
bad_data = PackData('', file=bad_file)
bad_pack = Pack.from_lazy_objects(lambda: bad_data, lambda: index)
self.assertRaises(ChecksumMismatch, lambda: bad_pack.data)
self.assertRaises(ChecksumMismatch, lambda:
bad_pack.check_length_and_checksum())
def test_thin_from_file(self):
test_sha = "1" * 40
def resolve(sha):
self.assertEqual(test_sha, sha)
return 3, "data"
path = os.path.join(self.datadir, "pack-%s.pack" % pack1_sha)
data = ThinPackData.from_file(resolve, open(path), os.path.getsize(path))
idx = self.get_pack_index(pack1_sha)
Pack.from_objects(data, idx)
self.assertEqual((None, 3, "data"), data.get_ref(test_sha))
def test_checksum_mismatch(self):
with self.get_pack_data(pack1_sha) as data:
index = self.get_pack_index(pack1_sha)
with Pack.from_objects(data, index) as p:
p.check_length_and_checksum()
data._file.seek(0)
with BytesIO(data._file.read()[:-20] + (b'\xff' * 20)) as bad_file:
with PackData('', file=bad_file) as bad_data:
with Pack.from_lazy_objects(lambda: bad_data, lambda: index) as bad_pack:
self.assertRaises(ChecksumMismatch, lambda: bad_pack.data)
self.assertRaises(ChecksumMismatch, lambda:
bad_pack.check_length_and_checksum())
def test_copy(self):
origpack = self.get_pack(pack1_sha)
self.assertEquals(True, origpack.idx.check())
write_pack("Elch", [(x, "") for x in origpack.iterobjects()], len(origpack))
newpack = Pack("Elch")
self.assertEquals(origpack, newpack)
self.assertEquals(True, newpack.idx.check())
self.assertEquals(origpack.name(), newpack.name())
self.assertEquals(origpack.idx.get_pack_checksum(), newpack.idx.get_pack_checksum())
self.assertTrue(
(origpack.idx.version != newpack.idx.version)
or (origpack.idx.get_stored_checksum() == newpack.idx.get_stored_checksum())
)
def test_length_mismatch(self):
data = self.get_pack_data(pack1_sha)
index = self.get_pack_index(pack1_sha)
Pack.from_objects(data, index).check_length_and_checksum()
data._file.seek(12)
bad_file = BytesIO()
write_pack_header(bad_file, 9999)
bad_file.write(data._file.read())
bad_file = BytesIO(bad_file.getvalue())
bad_data = PackData('', file=bad_file)
bad_pack = Pack.from_lazy_objects(lambda: bad_data, lambda: index)
self.assertRaises(AssertionError, lambda: bad_pack.data)
self.assertRaises(AssertionError,
lambda: bad_pack.check_length_and_checksum())
def test_length_mismatch(self):
with self.get_pack_data(pack1_sha) as data:
index = self.get_pack_index(pack1_sha)
with Pack.from_objects(data, index) as p:
p.check_length_and_checksum()
data._file.seek(12)
with BytesIO() as bad_file:
write_pack_header(bad_file, 9999)
bad_file.write(data._file.read())
with BytesIO(bad_file.getvalue()) as badder_file:
with PackData('', file=badder_file) as bad_data:
with Pack.from_lazy_objects(lambda: bad_data, lambda: index) as bad_pack:
self.assertRaises(AssertionError, lambda: bad_pack.data)
self.assertRaises(AssertionError,
lambda: bad_pack.check_length_and_checksum())
def move_in_thin_pack(self, path):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
data = PackData(path)
# Write index for the thin pack (do we really need this?)
temppath = os.path.join(self.pack_dir,
sha_to_hex(urllib2.randombytes(20))+".tempidx")
data.create_index_v2(temppath, self.get_raw)
p = Pack.from_objects(data, load_pack_index(temppath))
# Write a full pack version
temppath = os.path.join(self.pack_dir,
sha_to_hex(urllib2.randombytes(20))+".temppack")
write_pack(temppath, ((o, None) for o in p.iterobjects(self.get_raw)),
len(p))
pack_sha = load_pack_index(temppath+".idx").objects_sha1()
newbasename = os.path.join(self.pack_dir, "pack-%s" % pack_sha)
os.rename(temppath+".pack", newbasename+".pack")
os.rename(temppath+".idx", newbasename+".idx")
self._add_known_pack(newbasename)
def _create_pack(self, path):
def data_loader():
# read and writable temporary file
pack_tmpfile = tempfile.NamedTemporaryFile()
# download into temporary file
log.debug('Downloading pack %s into %s' % (path, pack_tmpfile))
pack_key = self.bucket.new_key('%s.pack' % path)
# store
pack_key.get_contents_to_file(pack_tmpfile)
log.debug('Filesize is %d' % pack_key.size)
log.debug('Rewinding...')
pack_tmpfile.flush()
pack_tmpfile.seek(0)
return PackData.from_file(pack_tmpfile, pack_key.size)
def idx_loader():
index_tmpfile = tempfile.NamedTemporaryFile()
log.debug('Downloading pack index %s into %s' % (path, index_tmpfile))
index_key = self.bucket.new_key('%s.idx' % path)
index_key.get_contents_to_file(index_tmpfile)
log.debug('Rewinding...')
index_tmpfile.flush()
index_tmpfile.seek(0)
return load_pack_index_file(index_tmpfile.name, index_tmpfile)
p = Pack(path)
p._data_load = data_loader
p._idx_load = idx_loader
return p
def pack_info_create(pack_data, pack_index):
pack = Pack.from_objects(pack_data, pack_index)
info = {}
for obj in pack.iterobjects():
# Commit
if obj.type_num == Commit.type_num:
info[obj.id] = (obj.type_num, obj.parents, obj.tree)
# Tree
elif obj.type_num == Tree.type_num:
shas = [(s, n, not stat.S_ISDIR(m)) for
n, m, s in obj.iteritems() if not S_ISGITLINK(m)]
info[obj.id] = (obj.type_num, shas)
# Blob
elif obj.type_num == Blob.type_num:
info[obj.id] = None
# Tag
elif obj.type_num == Tag.type_num:
info[obj.id] = (obj.type_num, obj.object[1])
return zlib.compress(json_dumps(info))
def get_pack(self, sha):
return Pack(os.path.join(self.datadir,
"pack-%s" % sha.decode("ascii")))
pack_dir_contents = os.listdir(self.pack_dir)
for name in pack_dir_contents:
# TODO: verify that idx exists first
if name.startswith("pack-") and name.endswith(".pack"):
filename = os.path.join(self.pack_dir, name)
pack_files.append((os.stat(filename).st_mtime, filename))
except OSError, e:
if e.errno == errno.ENOENT:
return []
raise
pack_files.sort(reverse=True)
suffix_len = len(".pack")
result = []
try:
for _, f in pack_files:
result.append(Pack(f[:-suffix_len]))
except:
for p in result:
p.close()
raise
return result
def _pack_cache_stale(self):
try:
return os.stat(self.pack_dir).st_mtime > self._pack_cache_time
except OSError, e:
if e.errno == errno.ENOENT:
return True
raise
def _get_shafile_path(self, sha):
def get_pack(self, sha):
return Pack(os.path.join(self.datadir, b'pack-' + sha))
pack_files = []
try:
self._pack_cache_time = os.stat(self.pack_dir).st_mtime
pack_dir_contents = os.listdir(self.pack_dir)
for name in pack_dir_contents:
# TODO: verify that idx exists first
if name.startswith("pack-") and name.endswith(".pack"):
filename = os.path.join(self.pack_dir, name)
pack_files.append((os.stat(filename).st_mtime, filename))
except OSError, e:
if e.errno == errno.ENOENT:
return []
raise
pack_files.sort(reverse=True)
suffix_len = len(".pack")
return [Pack(f[:-suffix_len]) for _, f in pack_files]
def _pack_cache_stale(self):
try:
return os.stat(self.pack_dir).st_mtime > self._pack_cache_time
except OSError, e:
if e.errno == errno.ENOENT:
return True
raise
def _get_shafile_path(self, sha):
# Check from object dir
return hex_to_filename(self.path, sha)
def _iter_loose_objects(self):
for base in os.listdir(self.path):
def _complete_thin_pack(self, f, path, copier, indexer):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param f: Open file object for the pack.
:param path: Path to the pack file.
:param copier: A PackStreamCopier to use for writing pack data.
:param indexer: A PackIndexer for indexing the pack.
"""
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.close()
# Move the pack in.
entries.sort()
pack_base_name = self._get_pack_basepath(entries)
target_pack = pack_base_name + '.pack'
if sys.platform == 'win32':
# Windows might have the target pack file lingering. Attempt
# removal, silently passing if the target does not exist.
try:
os.remove(target_pack)
except (IOError, OSError) as e:
if e.errno != errno.ENOENT:
raise
os.rename(path, target_pack)
# Write the index.
index_file = GitFile(pack_base_name + '.idx', 'wb')
try:
write_pack_index_v2(index_file, entries, pack_sha)
index_file.close()
finally:
index_file.abort()
# Add the pack to the store and return it.
final_pack = Pack(pack_base_name)
final_pack.check_length_and_checksum()
self._add_cached_pack(pack_base_name, final_pack)
return final_pack
def get_pack(self, sha):
return Pack(os.path.join(self.datadir, 'pack-%s' % sha.decode('ascii')))
def _complete_thin_pack(self, f, path, copier, indexer):
"""Move a specific file containing a pack into the pack directory.
Note: The file should be on the same file system as the
packs directory.
Args:
f: Open file object for the pack.
path: Path to the pack file.
copier: A PackStreamCopier to use for writing pack data.
indexer: A PackIndexer for indexing the pack.
"""
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(
f,
type_num,
data,
sha=new_sha,
compression_level=self.pack_compression_level)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.close()
# Move the pack in.
entries.sort()
pack_base_name = self._get_pack_basepath(entries)
target_pack = pack_base_name + '.pack'
if sys.platform == 'win32':
# Windows might have the target pack file lingering. Attempt
# removal, silently passing if the target does not exist.
try:
os.remove(target_pack)
except (IOError, OSError) as e:
if e.errno != errno.ENOENT:
raise
os.rename(path, target_pack)
# Write the index.
index_file = GitFile(pack_base_name + '.idx', 'wb')
try:
write_pack_index_v2(index_file, entries, pack_sha)
index_file.close()
finally:
index_file.abort()
# Add the pack to the store and return it.
final_pack = Pack(pack_base_name)
final_pack.check_length_and_checksum()
self._add_cached_pack(pack_base_name, final_pack)
return final_pack
def _copy_pack(self, origpack):
basename = os.path.join(self.tempdir, 'somepack')
write_pack(basename, origpack.pack_tuples())
return Pack(basename)
def get_objects():
pack = Pack(path[:-5])
for obj in pack.iterobjects():
yield obj
def _complete_thin_pack(self, f, path, copier, indexer):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param f: Open file object for the pack.
:param path: Path to the pack file.
:param copier: A PackStreamCopier to use for writing pack data.
:param indexer: A PackIndexer for indexing the pack.
"""
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.close()
# Move the pack in.
entries.sort()
pack_base_name = self._get_pack_basepath(entries)
if sys.platform == 'win32':
try:
os.rename(path, pack_base_name + '.pack')
except WindowsError:
os.remove(pack_base_name + '.pack')
os.rename(path, pack_base_name + '.pack')
else:
os.rename(path, pack_base_name + '.pack')
# Write the index.
index_file = GitFile(pack_base_name + '.idx', 'wb')
try:
write_pack_index_v2(index_file, entries, pack_sha)
index_file.close()
finally:
index_file.abort()
# Add the pack to the store and return it.
final_pack = Pack(pack_base_name)
final_pack.check_length_and_checksum()
self._add_known_pack(pack_base_name, final_pack)
return final_pack
def get_pack(self, sha):
return Pack(os.path.join(self.datadir, 'pack-%s' % sha))
def make_pack(self, resolve_ext_ref):
return Pack(
self.pack_prefix,
resolve_ext_ref=self.store.get_raw if resolve_ext_ref else None)
