def commit():
if pf.tell() == 0:
pf.close()
return None
pf.seek(0)
p = PackData(pf.name, pf)
entries = p.sorted_entries()
basename = iter_sha1(entry[0] for entry in entries).decode('ascii')
idxf = tempfile.SpooledTemporaryFile()
checksum = p.get_stored_checksum()
write_pack_index_v2(idxf, entries, checksum)
idxf.seek(0)
idx = load_pack_index_file(basename + '.idx', idxf)
for pack in self.packs:
if pack.get_stored_checksum() == p.get_stored_checksum():
p.close()
idx.close()
return pack
pf.seek(0)
idxf.seek(0)
self._upload_pack(basename, pf, idxf)
final_pack = Pack.from_objects(p, idx)
self._add_cached_pack(basename, final_pack)
return final_pack
def _update_pack_cache(self):
pack_files = set(self._pack_names())
new_packs = []
for basename in pack_files:
pack_name = basename + ".pack"
if basename not in self._pack_cache:
try:
size = self.pack_transport.stat(pack_name).st_size
except TransportNotPossible:
f = self.pack_transport.get(pack_name)
# TODO(jelmer): Don't read entire file into memory?
f = BytesIO(f.read())
pd = PackData(pack_name, f)
else:
pd = PackData(
pack_name, self.pack_transport.get(pack_name),
size=size)
idxname = basename + ".idx"
idx = load_pack_index_file(
idxname, self.pack_transport.get(idxname))
pack = Pack.from_objects(pd, idx)
pack._basename = basename
self._pack_cache[basename] = pack
new_packs.append(pack)
# Remove disappeared pack files
for f in set(self._pack_cache) - pack_files:
self._pack_cache.pop(f).close()
return new_packs
def move_in_pack(self, f):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
f.seek(0)
p = PackData("", f, len(f.getvalue()))
entries = p.sorted_entries()
basename = "pack-%s" % iter_sha1(entry[0]
for entry in entries).decode('ascii')
p._filename = basename + ".pack"
f.seek(0)
self.pack_transport.put_file(basename + ".pack", f)
idxfile = self.pack_transport.open_write_stream(basename + ".idx")
try:
write_pack_index_v2(idxfile, entries, p.get_stored_checksum())
finally:
idxfile.close()
idxfile = self.pack_transport.get(basename + ".idx")
idx = load_pack_index_file(basename + ".idx", idxfile)
final_pack = Pack.from_objects(p, idx)
final_pack._basename = basename
self._add_known_pack(basename, final_pack)
return final_pack
def _update_pack_cache(self):
pack_files = set()
pack_dir_contents = self._pack_names()
for name in pack_dir_contents:
if name.startswith("pack-") and name.endswith(".pack"):
# verify that idx exists first (otherwise the pack was not yet
# fully written)
idx_name = os.path.splitext(name)[0] + ".idx"
if idx_name in pack_dir_contents:
pack_files.add(os.path.splitext(name)[0])
new_packs = []
for basename in pack_files:
pack_name = basename + ".pack"
if basename not in self._pack_cache:
try:
size = self.pack_transport.stat(pack_name).st_size
except TransportNotPossible:
f = self.pack_transport.get(pack_name)
pd = PackData(pack_name, f)
else:
pd = PackData(pack_name,
self.pack_transport.get(pack_name),
size=size)
idxname = basename + ".idx"
idx = load_pack_index_file(idxname,
self.pack_transport.get(idxname))
pack = Pack.from_objects(pd, idx)
pack._basename = basename
self._pack_cache[basename] = pack
new_packs.append(pack)
# Remove disappeared pack files
for f in set(self._pack_cache) - pack_files:
self._pack_cache.pop(f).close()
return new_packs
def _complete_thin_pack(self, f, path, copier, indexer):
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.flush()
# Move the pack in.
entries.sort()
pack_base_name = posixpath.join(
self.pack_dir,
"pack-" + os.fsdecode(iter_sha1(e[0] for e in entries)),
)
self.scon.put_object(pack_base_name + ".pack", f)
# Write the index.
filename = pack_base_name + ".idx"
index_file = BytesIO()
write_pack_index_v2(index_file, entries, pack_sha)
self.scon.put_object(filename, index_file)
# Write pack info.
f.seek(0)
pack_data = PackData(filename="", file=f)
index_file.seek(0)
pack_index = load_pack_index_file("", index_file)
serialized_pack_info = pack_info_create(pack_data, pack_index)
f.close()
index_file.close()
pack_info_file = BytesIO(serialized_pack_info)
filename = pack_base_name + ".info"
self.scon.put_object(filename, pack_info_file)
pack_info_file.close()
# Add the pack to the store and return it.
final_pack = SwiftPack(pack_base_name, scon=self.scon)
final_pack.check_length_and_checksum()
self._add_cached_pack(pack_base_name, final_pack)
return final_pack
def _complete_thin_pack(self, f, path, copier, indexer):
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.flush()
# Move the pack in.
entries.sort()
pack_base_name = posixpath.join(
self.pack_dir,
'pack-' + iter_sha1(e[0] for e in entries).decode(
sys.getfilesystemencoding()))
self.scon.put_object(pack_base_name + '.pack', f)
# Write the index.
filename = pack_base_name + '.idx'
index_file = BytesIO()
write_pack_index_v2(index_file, entries, pack_sha)
self.scon.put_object(filename, index_file)
# Write pack info.
f.seek(0)
pack_data = PackData(filename="", file=f)
index_file.seek(0)
pack_index = load_pack_index_file('', index_file)
serialized_pack_info = pack_info_create(pack_data, pack_index)
f.close()
index_file.close()
pack_info_file = BytesIO(serialized_pack_info)
filename = pack_base_name + '.info'
self.scon.put_object(filename, pack_info_file)
pack_info_file.close()
# Add the pack to the store and return it.
final_pack = SwiftPack(pack_base_name, scon=self.scon)
final_pack.check_length_and_checksum()
self._add_cached_pack(pack_base_name, final_pack)
return final_pack
def swift_load_pack_index(scon, filename):
"""Read a pack index file from Swift
:param scon: a `SwiftConnector` instance
:param filename: Path to the index file objectise
:return: a `PackIndexer` instance
"""
with scon.get_object(filename) as f:
return load_pack_index_file(filename, f)
def idx_loader():
index_tmpfile = tempfile.NamedTemporaryFile()
log.debug('Downloading pack index %s into %s' % (path, index_tmpfile))
index_key = self.bucket.new_key('%s.idx' % path)
index_key.get_contents_to_file(index_tmpfile)
log.debug('Rewinding...')
index_tmpfile.flush()
index_tmpfile.seek(0)
return load_pack_index_file(index_tmpfile.name, index_tmpfile)
def setUp(self):
conf = swift.load_conf(file=StringIO(config_file % def_config_file))
sos = swift.SwiftObjectStore(FakeSwiftConnector("fakerepo", conf=conf))
commit_amount = 10
self.commits = create_commits(length=commit_amount, marker="m")
data = [(d.type_num, d.as_raw_string()) for d in self.commits]
f = StringIO()
fi = StringIO()
expected = build_pack(f, data, store=sos)
entries = [(sha, ofs, checksum) for ofs, _, _, sha, checksum in expected]
self.pack_data = PackData.from_file(file=f, size=None)
write_pack_index_v2(fi, entries, self.pack_data.calculate_checksum())
fi.seek(0)
self.pack_index = load_pack_index_file("", fi)
def setUp(self):
conf = swift.load_conf(file=BytesIO(config_file % def_config_file))
sos = swift.SwiftObjectStore(FakeSwiftConnector('fakerepo', conf=conf))
commit_amount = 10
self.commits = create_commits(length=commit_amount, marker="m")
data = [(d.type_num, d.as_raw_string()) for d in self.commits]
f = BytesIO()
fi = BytesIO()
expected = build_pack(f, data, store=sos)
entries = [(sha, ofs, checksum)
for ofs, _, _, sha, checksum in expected]
self.pack_data = PackData.from_file(file=f, size=None)
write_pack_index_v2(fi, entries, self.pack_data.calculate_checksum())
fi.seek(0)
self.pack_index = load_pack_index_file('', fi)
def _load_packs(self):
ret = []
for name in self._pack_names():
if name.startswith("pack-") and name.endswith(".pack"):
try:
size = self.pack_transport.stat(name).st_size
except TransportNotPossible:
f = self.pack_transport.get(name)
pd = PackData(name, f)
else:
pd = PackData(name,
self.pack_transport.get(name),
size=size)
idxname = name.replace(".pack", ".idx")
idx = load_pack_index_file(idxname,
self.pack_transport.get(idxname))
pack = Pack.from_objects(pd, idx)
pack._basename = idxname[:-4]
ret.append(pack)
return ret
def _pack_idx() -> PackIndex:
pack_idx_bytes = self._http_get(f"objects/pack/{pack_idx_name}")
return load_pack_index_file(pack_idx_name, pack_idx_bytes)
