def test_compute_file_sha(self):
f = BytesIO('abcd1234wxyz')
self.assertEqual(sha1('abcd1234wxyz').hexdigest(),
compute_file_sha(f).hexdigest())
self.assertEqual(sha1('abcd1234wxyz').hexdigest(),
compute_file_sha(f, buffer_size=5).hexdigest())
self.assertEqual(sha1('abcd1234').hexdigest(),
compute_file_sha(f, end_ofs=-4).hexdigest())
self.assertEqual(sha1('1234wxyz').hexdigest(),
compute_file_sha(f, start_ofs=4).hexdigest())
self.assertEqual(
sha1('1234').hexdigest(),
compute_file_sha(f, start_ofs=4, end_ofs=-4).hexdigest())
def test_compute_file_sha(self):
f = BytesIO('abcd1234wxyz')
self.assertEqual(sha1('abcd1234wxyz').hexdigest(),
compute_file_sha(f).hexdigest())
self.assertEqual(sha1('abcd1234wxyz').hexdigest(),
compute_file_sha(f, buffer_size=5).hexdigest())
self.assertEqual(sha1('abcd1234').hexdigest(),
compute_file_sha(f, end_ofs=-4).hexdigest())
self.assertEqual(sha1('1234wxyz').hexdigest(),
compute_file_sha(f, start_ofs=4).hexdigest())
self.assertEqual(
sha1('1234').hexdigest(),
compute_file_sha(f, start_ofs=4, end_ofs=-4).hexdigest())
def test_compute_file_sha(self):
with BytesIO(b'abcd1234wxyz') as f:
self.assertEqual(hashlib.sha1(b'abcd1234wxyz').hexdigest(),
compute_file_sha(f).hexdigest())
self.assertEqual(hashlib.sha1(b'abcd1234wxyz').hexdigest(),
compute_file_sha(f, buffer_size=5).hexdigest())
self.assertEqual(hashlib.sha1(b'abcd1234').hexdigest(),
compute_file_sha(f, end_ofs=-4).hexdigest())
self.assertEqual(hashlib.sha1(b'1234wxyz').hexdigest(),
compute_file_sha(f, start_ofs=4).hexdigest())
self.assertEqual(
hashlib.sha1(b'1234').hexdigest(),
compute_file_sha(f, start_ofs=4, end_ofs=-4).hexdigest())
def _complete_thin_pack(self, f, path, copier, indexer):
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.flush()
# Move the pack in.
entries.sort()
pack_base_name = posixpath.join(
self.pack_dir,
"pack-" + os.fsdecode(iter_sha1(e[0] for e in entries)),
)
self.scon.put_object(pack_base_name + ".pack", f)
# Write the index.
filename = pack_base_name + ".idx"
index_file = BytesIO()
write_pack_index_v2(index_file, entries, pack_sha)
self.scon.put_object(filename, index_file)
# Write pack info.
f.seek(0)
pack_data = PackData(filename="", file=f)
index_file.seek(0)
pack_index = load_pack_index_file("", index_file)
serialized_pack_info = pack_info_create(pack_data, pack_index)
f.close()
index_file.close()
pack_info_file = BytesIO(serialized_pack_info)
filename = pack_base_name + ".info"
self.scon.put_object(filename, pack_info_file)
pack_info_file.close()
# Add the pack to the store and return it.
final_pack = SwiftPack(pack_base_name, scon=self.scon)
final_pack.check_length_and_checksum()
self._add_cached_pack(pack_base_name, final_pack)
return final_pack
def _complete_thin_pack(self, f, path, copier, indexer):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param f: Open file object for the pack.
:param path: Path to the pack file.
:param copier: A PackStreamCopier to use for writing pack data.
:param indexer: A PackIndexer for indexing the pack.
"""
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.close()
# Move the pack in.
entries.sort()
pack_base_name = self._get_pack_basepath(entries)
try:
os.rename(path, pack_base_name + '.pack')
except WindowsError:
os.remove(pack_base_name + '.pack')
os.rename(path, pack_base_name + '.pack')
# Write the index.
index_file = GitFile(pack_base_name + '.idx', 'wb')
try:
write_pack_index_v2(index_file, entries, pack_sha)
index_file.close()
finally:
index_file.abort()
# Add the pack to the store and return it.
final_pack = Pack(pack_base_name)
final_pack.check_length_and_checksum()
self._add_known_pack(pack_base_name, final_pack)
return final_pack
def _complete_thin_pack(self, f, path, copier, indexer):
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.flush()
# Move the pack in.
entries.sort()
pack_base_name = posixpath.join(
self.pack_dir,
'pack-' + iter_sha1(e[0] for e in entries).decode(
sys.getfilesystemencoding()))
self.scon.put_object(pack_base_name + '.pack', f)
# Write the index.
filename = pack_base_name + '.idx'
index_file = BytesIO()
write_pack_index_v2(index_file, entries, pack_sha)
self.scon.put_object(filename, index_file)
# Write pack info.
f.seek(0)
pack_data = PackData(filename="", file=f)
index_file.seek(0)
pack_index = load_pack_index_file('', index_file)
serialized_pack_info = pack_info_create(pack_data, pack_index)
f.close()
index_file.close()
pack_info_file = BytesIO(serialized_pack_info)
filename = pack_base_name + '.info'
self.scon.put_object(filename, pack_info_file)
pack_info_file.close()
# Add the pack to the store and return it.
final_pack = SwiftPack(pack_base_name, scon=self.scon)
final_pack.check_length_and_checksum()
self._add_cached_pack(pack_base_name, final_pack)
return final_pack
def _complete_thin_pack(self, f, path, copier, indexer):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param f: Open file object for the pack.
:param path: Path to the pack file.
:param copier: A PackStreamCopier to use for writing pack data.
:param indexer: A PackIndexer for indexing the pack.
"""
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.close()
# Move the pack in.
entries.sort()
pack_base_name = os.path.join(
self.pack_dir, 'pack-' + iter_sha1(e[0] for e in entries))
os.rename(path, pack_base_name + '.pack')
# Write the index.
index_file = GitFile(pack_base_name + '.idx', 'wb')
try:
write_pack_index_v2(index_file, entries, pack_sha)
index_file.close()
finally:
index_file.abort()
# Add the pack to the store and return it.
final_pack = Pack(pack_base_name)
final_pack.check_length_and_checksum()
self._add_known_pack(final_pack)
return final_pack
def from_thinpack(cls, pack_store, f, indexer, resolve_ext_ref):
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = resolve_ext_ref(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
#f.close()
#write the pack
blob_name = files.blobstore.create(
mime_type='application/octet-stream')
with files.open(blob_name, 'a') as blob:
blob.write(f.getvalue())
files.finalize(blob_name)
#store pack info
pack_store.data = files.blobstore.get_blob_key(blob_name)
#pack_store.sha1 #sha's are added when writing the index
pack_store.size = f.tell()
pack_store.checksum = sha_to_hex(pack_sha)
pack_store.save()
# Write the index.
pack_indexes = [pack_store]
for (name, offset, entry_checksum) in entries:
idx = PackStoreIndex(packref=pack_store,
sha=sha_to_hex(name),
offset=offset,
crc32=entry_checksum)
pack_store.sha1.append(sha_to_hex(name))
pack_indexes.append(idx)
db.save(pack_indexes)
# Add the pack to the store and return it.
final_pack = Pack(pack_store)
final_pack.check_length_and_checksum()
return final_pack
def from_thinpack(cls, pack_store, f, indexer, resolve_ext_ref):
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = resolve_ext_ref(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
# f.close()
# write the pack
blob_name = files.blobstore.create(mime_type="application/octet-stream")
with files.open(blob_name, "a") as blob:
blob.write(f.getvalue())
files.finalize(blob_name)
# store pack info
pack_store.data = files.blobstore.get_blob_key(blob_name)
# pack_store.sha1 #sha's are added when writing the index
pack_store.size = f.tell()
pack_store.checksum = sha_to_hex(pack_sha)
pack_store.save()
# Write the index.
pack_indexes = [pack_store]
for (name, offset, entry_checksum) in entries:
idx = PackStoreIndex(packref=pack_store, sha=sha_to_hex(name), offset=offset, crc32=entry_checksum)
pack_store.sha1.append(sha_to_hex(name))
pack_indexes.append(idx)
db.save(pack_indexes)
# Add the pack to the store and return it.
final_pack = Pack(pack_store)
final_pack.check_length_and_checksum()
return final_pack
def _complete_thin_pack(self, f, indexer):
"""Complete a thin pack by adding external references.
:param f: Open file object for the pack.
:param indexer: A PackIndexer for indexing the pack.
"""
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
write_pack_object(f, type_num, data, sha=new_sha)
pack_sha = new_sha.digest()
f.write(pack_sha)
def _complete_thin_pack(self, f, indexer):
"""Complete a thin pack by adding external references.
:param f: Open file object for the pack.
:param indexer: A PackIndexer for indexing the pack.
"""
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
write_pack_object(f, type_num, data, sha=new_sha)
pack_sha = new_sha.digest()
f.write(pack_sha)
def _complete_thin_pack(self, f, path, copier, indexer):
"""Move a specific file containing a pack into the pack directory.
Note: The file should be on the same file system as the
packs directory.
Args:
f: Open file object for the pack.
path: Path to the pack file.
copier: A PackStreamCopier to use for writing pack data.
indexer: A PackIndexer for indexing the pack.
"""
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(
f,
type_num,
data,
sha=new_sha,
compression_level=self.pack_compression_level,
)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.close()
# Move the pack in.
entries.sort()
pack_base_name = self._get_pack_basepath(entries)
target_pack = pack_base_name + ".pack"
if sys.platform == "win32":
# Windows might have the target pack file lingering. Attempt
# removal, silently passing if the target does not exist.
try:
os.remove(target_pack)
except FileNotFoundError:
pass
os.rename(path, target_pack)
# Write the index.
index_file = GitFile(pack_base_name + ".idx", "wb")
try:
write_pack_index_v2(index_file, entries, pack_sha)
index_file.close()
finally:
index_file.abort()
# Add the pack to the store and return it.
final_pack = Pack(pack_base_name)
final_pack.check_length_and_checksum()
self._add_cached_pack(pack_base_name, final_pack)
return final_pack
def _complete_thin_pack(self, f, path, copier, indexer):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param f: Open file object for the pack.
:param path: Path to the pack file.
:param copier: A PackStreamCopier to use for writing pack data.
:param indexer: A PackIndexer for indexing the pack.
"""
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.close()
# Move the pack in.
entries.sort()
pack_base_name = self._get_pack_basepath(entries)
target_pack = pack_base_name + '.pack'
if sys.platform == 'win32':
# Windows might have the target pack file lingering. Attempt
# removal, silently passing if the target does not exist.
try:
os.remove(target_pack)
except (IOError, OSError) as e:
if e.errno != errno.ENOENT:
raise
os.rename(path, target_pack)
# Write the index.
index_file = GitFile(pack_base_name + '.idx', 'wb')
try:
write_pack_index_v2(index_file, entries, pack_sha)
index_file.close()
finally:
index_file.abort()
# Add the pack to the store and return it.
final_pack = Pack(pack_base_name)
final_pack.check_length_and_checksum()
self._add_cached_pack(pack_base_name, final_pack)
return final_pack