def _complete_thin_pack(self, f, path, copier, indexer):
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.flush()
# Move the pack in.
entries.sort()
pack_base_name = posixpath.join(
self.pack_dir,
"pack-" + os.fsdecode(iter_sha1(e[0] for e in entries)),
)
self.scon.put_object(pack_base_name + ".pack", f)
# Write the index.
filename = pack_base_name + ".idx"
index_file = BytesIO()
write_pack_index_v2(index_file, entries, pack_sha)
self.scon.put_object(filename, index_file)
# Write pack info.
f.seek(0)
pack_data = PackData(filename="", file=f)
index_file.seek(0)
pack_index = load_pack_index_file("", index_file)
serialized_pack_info = pack_info_create(pack_data, pack_index)
f.close()
index_file.close()
pack_info_file = BytesIO(serialized_pack_info)
filename = pack_base_name + ".info"
self.scon.put_object(filename, pack_info_file)
pack_info_file.close()
# Add the pack to the store and return it.
final_pack = SwiftPack(pack_base_name, scon=self.scon)
final_pack.check_length_and_checksum()
self._add_cached_pack(pack_base_name, final_pack)
return final_pack
def _complete_thin_pack(self, f, path, copier, indexer):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param f: Open file object for the pack.
:param path: Path to the pack file.
:param copier: A PackStreamCopier to use for writing pack data.
:param indexer: A PackIndexer for indexing the pack.
"""
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.close()
# Move the pack in.
entries.sort()
pack_base_name = self._get_pack_basepath(entries)
try:
os.rename(path, pack_base_name + '.pack')
except WindowsError:
os.remove(pack_base_name + '.pack')
os.rename(path, pack_base_name + '.pack')
# Write the index.
index_file = GitFile(pack_base_name + '.idx', 'wb')
try:
write_pack_index_v2(index_file, entries, pack_sha)
index_file.close()
finally:
index_file.abort()
# Add the pack to the store and return it.
final_pack = Pack(pack_base_name)
final_pack.check_length_and_checksum()
self._add_known_pack(pack_base_name, final_pack)
return final_pack
def _complete_thin_pack(self, f, path, copier, indexer):
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.flush()
# Move the pack in.
entries.sort()
pack_base_name = posixpath.join(
self.pack_dir,
'pack-' + iter_sha1(e[0] for e in entries).decode(
sys.getfilesystemencoding()))
self.scon.put_object(pack_base_name + '.pack', f)
# Write the index.
filename = pack_base_name + '.idx'
index_file = BytesIO()
write_pack_index_v2(index_file, entries, pack_sha)
self.scon.put_object(filename, index_file)
# Write pack info.
f.seek(0)
pack_data = PackData(filename="", file=f)
index_file.seek(0)
pack_index = load_pack_index_file('', index_file)
serialized_pack_info = pack_info_create(pack_data, pack_index)
f.close()
index_file.close()
pack_info_file = BytesIO(serialized_pack_info)
filename = pack_base_name + '.info'
self.scon.put_object(filename, pack_info_file)
pack_info_file.close()
# Add the pack to the store and return it.
final_pack = SwiftPack(pack_base_name, scon=self.scon)
final_pack.check_length_and_checksum()
self._add_cached_pack(pack_base_name, final_pack)
return final_pack
def _complete_thin_pack(self, f, path, copier, indexer):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param f: Open file object for the pack.
:param path: Path to the pack file.
:param copier: A PackStreamCopier to use for writing pack data.
:param indexer: A PackIndexer for indexing the pack.
"""
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.close()
# Move the pack in.
entries.sort()
pack_base_name = os.path.join(
self.pack_dir, 'pack-' + iter_sha1(e[0] for e in entries))
os.rename(path, pack_base_name + '.pack')
# Write the index.
index_file = GitFile(pack_base_name + '.idx', 'wb')
try:
write_pack_index_v2(index_file, entries, pack_sha)
index_file.close()
finally:
index_file.abort()
# Add the pack to the store and return it.
final_pack = Pack(pack_base_name)
final_pack.check_length_and_checksum()
self._add_known_pack(final_pack)
return final_pack
def from_thinpack(cls, pack_store, f, indexer, resolve_ext_ref):
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = resolve_ext_ref(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
#f.close()
#write the pack
blob_name = files.blobstore.create(
mime_type='application/octet-stream')
with files.open(blob_name, 'a') as blob:
blob.write(f.getvalue())
files.finalize(blob_name)
#store pack info
pack_store.data = files.blobstore.get_blob_key(blob_name)
#pack_store.sha1 #sha's are added when writing the index
pack_store.size = f.tell()
pack_store.checksum = sha_to_hex(pack_sha)
pack_store.save()
# Write the index.
pack_indexes = [pack_store]
for (name, offset, entry_checksum) in entries:
idx = PackStoreIndex(packref=pack_store,
sha=sha_to_hex(name),
offset=offset,
crc32=entry_checksum)
pack_store.sha1.append(sha_to_hex(name))
pack_indexes.append(idx)
db.save(pack_indexes)
# Add the pack to the store and return it.
final_pack = Pack(pack_store)
final_pack.check_length_and_checksum()
return final_pack
def test_length_mismatch(self):
data = self.get_pack_data(pack1_sha)
index = self.get_pack_index(pack1_sha)
Pack.from_objects(data, index).check_length_and_checksum()
data._file.seek(12)
bad_file = BytesIO()
write_pack_header(bad_file, 9999)
bad_file.write(data._file.read())
bad_file = BytesIO(bad_file.getvalue())
bad_data = PackData('', file=bad_file)
bad_pack = Pack.from_lazy_objects(lambda: bad_data, lambda: index)
self.assertRaises(AssertionError, lambda: bad_pack.data)
self.assertRaises(AssertionError,
lambda: bad_pack.check_length_and_checksum())
def test_length_mismatch(self):
with self.get_pack_data(pack1_sha) as data:
index = self.get_pack_index(pack1_sha)
Pack.from_objects(data, index).check_length_and_checksum()
data._file.seek(12)
bad_file = BytesIO()
write_pack_header(bad_file, 9999)
bad_file.write(data._file.read())
bad_file = BytesIO(bad_file.getvalue())
bad_data = PackData('', file=bad_file)
bad_pack = Pack.from_lazy_objects(lambda: bad_data, lambda: index)
self.assertRaises(AssertionError, lambda: bad_pack.data)
self.assertRaises(AssertionError,
lambda: bad_pack.check_length_and_checksum())
def test_length_mismatch(self):
with self.get_pack_data(pack1_sha) as data:
index = self.get_pack_index(pack1_sha)
with Pack.from_objects(data, index) as p:
p.check_length_and_checksum()
data._file.seek(12)
with BytesIO() as bad_file:
write_pack_header(bad_file, 9999)
bad_file.write(data._file.read())
with BytesIO(bad_file.getvalue()) as badder_file:
with PackData('', file=badder_file) as bad_data:
with Pack.from_lazy_objects(lambda: bad_data, lambda: index) as bad_pack:
self.assertRaises(AssertionError, lambda: bad_pack.data)
self.assertRaises(AssertionError,
lambda: bad_pack.check_length_and_checksum())
def from_thinpack(cls, pack_store, f, indexer, resolve_ext_ref):
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = resolve_ext_ref(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
# f.close()
# write the pack
blob_name = files.blobstore.create(mime_type="application/octet-stream")
with files.open(blob_name, "a") as blob:
blob.write(f.getvalue())
files.finalize(blob_name)
# store pack info
pack_store.data = files.blobstore.get_blob_key(blob_name)
# pack_store.sha1 #sha's are added when writing the index
pack_store.size = f.tell()
pack_store.checksum = sha_to_hex(pack_sha)
pack_store.save()
# Write the index.
pack_indexes = [pack_store]
for (name, offset, entry_checksum) in entries:
idx = PackStoreIndex(packref=pack_store, sha=sha_to_hex(name), offset=offset, crc32=entry_checksum)
pack_store.sha1.append(sha_to_hex(name))
pack_indexes.append(idx)
db.save(pack_indexes)
# Add the pack to the store and return it.
final_pack = Pack(pack_store)
final_pack.check_length_and_checksum()
return final_pack
def _complete_thin_pack(self, f, indexer):
"""Complete a thin pack by adding external references.
:param f: Open file object for the pack.
:param indexer: A PackIndexer for indexing the pack.
"""
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
write_pack_object(f, type_num, data, sha=new_sha)
pack_sha = new_sha.digest()
f.write(pack_sha)
def _complete_thin_pack(self, f, indexer):
"""Complete a thin pack by adding external references.
:param f: Open file object for the pack.
:param indexer: A PackIndexer for indexing the pack.
"""
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
write_pack_object(f, type_num, data, sha=new_sha)
pack_sha = new_sha.digest()
f.write(pack_sha)
def test_write_pack_header(self):
f = BytesIO()
write_pack_header(f, 42)
self.assertEqual('PACK\x00\x00\x00\x02\x00\x00\x00*',
f.getvalue())
def _complete_thin_pack(self, f, path, copier, indexer):
"""Move a specific file containing a pack into the pack directory.
Note: The file should be on the same file system as the
packs directory.
Args:
f: Open file object for the pack.
path: Path to the pack file.
copier: A PackStreamCopier to use for writing pack data.
indexer: A PackIndexer for indexing the pack.
"""
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(
f,
type_num,
data,
sha=new_sha,
compression_level=self.pack_compression_level,
)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.close()
# Move the pack in.
entries.sort()
pack_base_name = self._get_pack_basepath(entries)
target_pack = pack_base_name + ".pack"
if sys.platform == "win32":
# Windows might have the target pack file lingering. Attempt
# removal, silently passing if the target does not exist.
try:
os.remove(target_pack)
except FileNotFoundError:
pass
os.rename(path, target_pack)
# Write the index.
index_file = GitFile(pack_base_name + ".idx", "wb")
try:
write_pack_index_v2(index_file, entries, pack_sha)
index_file.close()
finally:
index_file.abort()
# Add the pack to the store and return it.
final_pack = Pack(pack_base_name)
final_pack.check_length_and_checksum()
self._add_cached_pack(pack_base_name, final_pack)
return final_pack
def test_simple(self):
f = StringIO()
write_pack_header(f, 42)
self.assertEquals("PACK\x00\x00\x00\x02\x00\x00\x00*", f.getvalue())
def build_pack(f, objects_spec, store=None):
"""Write test pack data from a concise spec.
Args:
f: A file-like object to write the pack to.
objects_spec: A list of (type_num, obj). For non-delta types, obj
is the string of that object's data.
For delta types, obj is a tuple of (base, data), where:
* base can be either an index in objects_spec of the base for that
* delta; or for a ref delta, a SHA, in which case the resulting pack
* will be thin and the base will be an external ref.
* data is a string of the full, non-deltified data for that object.
Note that offsets/refs and deltas are computed within this function.
store: An optional ObjectStore for looking up external refs.
Returns: A list of tuples in the order specified by objects_spec:
(offset, type num, data, sha, CRC32)
"""
sf = SHA1Writer(f)
num_objects = len(objects_spec)
write_pack_header(sf, num_objects)
full_objects = {}
offsets = {}
crc32s = {}
while len(full_objects) < num_objects:
for i, (type_num, data) in enumerate(objects_spec):
if type_num not in DELTA_TYPES:
full_objects[i] = (type_num, data, obj_sha(type_num, [data]))
continue
base, data = data
if isinstance(base, int):
if base not in full_objects:
continue
base_type_num, _, _ = full_objects[base]
else:
base_type_num, _ = store.get_raw(base)
full_objects[i] = (
base_type_num,
data,
obj_sha(base_type_num, [data]),
)
for i, (type_num, obj) in enumerate(objects_spec):
offset = f.tell()
if type_num == OFS_DELTA:
base_index, data = obj
base = offset - offsets[base_index]
_, base_data, _ = full_objects[base_index]
obj = (base, create_delta(base_data, data))
elif type_num == REF_DELTA:
base_ref, data = obj
if isinstance(base_ref, int):
_, base_data, base = full_objects[base_ref]
else:
base_type_num, base_data = store.get_raw(base_ref)
base = obj_sha(base_type_num, base_data)
obj = (base, create_delta(base_data, data))
crc32 = write_pack_object(sf, type_num, obj)
offsets[i] = offset
crc32s[i] = crc32
expected = []
for i in range(num_objects):
type_num, data, sha = full_objects[i]
assert len(sha) == 20
expected.append((offsets[i], type_num, data, sha, crc32s[i]))
sf.write_sha()
f.seek(0)
return expected
def build_pack(f, objects_spec, store=None):
"""Write test pack data from a concise spec.
:param f: A file-like object to write the pack to.
:param objects_spec: A list of (type_num, obj). For non-delta types, obj
is the string of that object's data.
For delta types, obj is a tuple of (base, data), where:
* base can be either an index in objects_spec of the base for that
* delta; or for a ref delta, a SHA, in which case the resulting pack
* will be thin and the base will be an external ref.
* data is a string of the full, non-deltified data for that object.
Note that offsets/refs and deltas are computed within this function.
:param store: An optional ObjectStore for looking up external refs.
:return: A list of tuples in the order specified by objects_spec:
(offset, type num, data, sha, CRC32)
"""
sf = SHA1Writer(f)
num_objects = len(objects_spec)
write_pack_header(sf, num_objects)
full_objects = {}
offsets = {}
crc32s = {}
while len(full_objects) < num_objects:
for i, (type_num, data) in enumerate(objects_spec):
if type_num not in DELTA_TYPES:
full_objects[i] = (type_num, data,
obj_sha(type_num, [data]))
continue
base, data = data
if isinstance(base, int):
if base not in full_objects:
continue
base_type_num, _, _ = full_objects[base]
else:
base_type_num, _ = store.get_raw(base)
full_objects[i] = (base_type_num, data,
obj_sha(base_type_num, [data]))
for i, (type_num, obj) in enumerate(objects_spec):
offset = f.tell()
if type_num == OFS_DELTA:
base_index, data = obj
base = offset - offsets[base_index]
_, base_data, _ = full_objects[base_index]
obj = (base, create_delta(base_data, data))
elif type_num == REF_DELTA:
base_ref, data = obj
if isinstance(base_ref, int):
_, base_data, base = full_objects[base_ref]
else:
base_type_num, base_data = store.get_raw(base_ref)
base = obj_sha(base_type_num, base_data)
obj = (base, create_delta(base_data, data))
crc32 = write_pack_object(sf, type_num, obj)
offsets[i] = offset
crc32s[i] = crc32
expected = []
for i in range(num_objects):
type_num, data, sha = full_objects[i]
expected.append((offsets[i], type_num, data, sha, crc32s[i]))
sf.write_sha()
f.seek(0)
return expected
def test_write_pack_header(self):
f = BytesIO()
write_pack_header(f, 42)
self.assertEqual('PACK\x00\x00\x00\x02\x00\x00\x00*',
f.getvalue())
def test_write_pack_header(self):
with BytesIO() as f:
write_pack_header(f, 42)
self.assertEqual(b'PACK\x00\x00\x00\x02\x00\x00\x00*',
f.getvalue())
def _complete_thin_pack(self, f, path, copier, indexer):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param f: Open file object for the pack.
:param path: Path to the pack file.
:param copier: A PackStreamCopier to use for writing pack data.
:param indexer: A PackIndexer for indexing the pack.
"""
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.close()
# Move the pack in.
entries.sort()
pack_base_name = self._get_pack_basepath(entries)
target_pack = pack_base_name + '.pack'
if sys.platform == 'win32':
# Windows might have the target pack file lingering. Attempt
# removal, silently passing if the target does not exist.
try:
os.remove(target_pack)
except (IOError, OSError) as e:
if e.errno != errno.ENOENT:
raise
os.rename(path, target_pack)
# Write the index.
index_file = GitFile(pack_base_name + '.idx', 'wb')
try:
write_pack_index_v2(index_file, entries, pack_sha)
index_file.close()
finally:
index_file.abort()
# Add the pack to the store and return it.
final_pack = Pack(pack_base_name)
final_pack.check_length_and_checksum()
self._add_cached_pack(pack_base_name, final_pack)
return final_pack
