def _update_pack_cache(self):
pack_files = set()
pack_dir_contents = self._pack_names()
for name in pack_dir_contents:
if name.startswith("pack-") and name.endswith(".pack"):
# verify that idx exists first (otherwise the pack was not yet
# fully written)
idx_name = os.path.splitext(name)[0] + ".idx"
if idx_name in pack_dir_contents:
pack_files.add(os.path.splitext(name)[0])
new_packs = []
for basename in pack_files:
pack_name = basename + ".pack"
if basename not in self._pack_cache:
try:
size = self.pack_transport.stat(pack_name).st_size
except TransportNotPossible:
f = self.pack_transport.get(pack_name)
pd = PackData(pack_name, f)
else:
pd = PackData(pack_name,
self.pack_transport.get(pack_name),
size=size)
idxname = basename + ".idx"
idx = load_pack_index_file(idxname,
self.pack_transport.get(idxname))
pack = Pack.from_objects(pd, idx)
pack._basename = basename
self._pack_cache[basename] = pack
new_packs.append(pack)
# Remove disappeared pack files
for f in set(self._pack_cache) - pack_files:
self._pack_cache.pop(f).close()
return new_packs
def _update_pack_cache(self):
pack_files = set(self._pack_names())
new_packs = []
for basename in pack_files:
pack_name = basename + ".pack"
if basename not in self._pack_cache:
try:
size = self.pack_transport.stat(pack_name).st_size
except TransportNotPossible:
f = self.pack_transport.get(pack_name)
# TODO(jelmer): Don't read entire file into memory?
f = BytesIO(f.read())
pd = PackData(pack_name, f)
else:
pd = PackData(
pack_name, self.pack_transport.get(pack_name),
size=size)
idxname = basename + ".idx"
idx = load_pack_index_file(
idxname, self.pack_transport.get(idxname))
pack = Pack.from_objects(pd, idx)
pack._basename = basename
self._pack_cache[basename] = pack
new_packs.append(pack)
# Remove disappeared pack files
for f in set(self._pack_cache) - pack_files:
self._pack_cache.pop(f).close()
return new_packs
def move_in_pack(self, path):
"""Move a specific file containing a pack into the pack directory.
Note: The file should be on the same file system as the
packs directory.
Args:
path: Path to the pack file.
"""
with PackData(path) as p:
entries = p.sorted_entries()
basename = self._get_pack_basepath(entries)
index_name = basename + ".idx"
if not os.path.exists(index_name):
with GitFile(index_name, "wb") as f:
write_pack_index_v2(f, entries, p.get_stored_checksum())
for pack in self.packs:
if pack._basename == basename:
return pack
target_pack = basename + ".pack"
if sys.platform == "win32":
# Windows might have the target pack file lingering. Attempt
# removal, silently passing if the target does not exist.
try:
os.remove(target_pack)
except FileNotFoundError:
pass
os.rename(path, target_pack)
final_pack = Pack(basename)
self._add_cached_pack(basename, final_pack)
return final_pack
def setUp(self):
super(TestThinPack, self).setUp()
self.store = MemoryObjectStore()
self.blobs = {}
for blob in (b'foo', b'bar', b'foo1234', b'bar2468'):
self.blobs[blob] = make_object(Blob, data=blob)
self.store.add_object(self.blobs[b'foo'])
self.store.add_object(self.blobs[b'bar'])
# Build a thin pack. 'foo' is as an external reference, 'bar' an
# internal reference.
self.pack_dir = tempfile.mkdtemp()
if not isinstance(self.pack_dir, bytes):
self.pack_dir = self.pack_dir.encode(sys.getfilesystemencoding())
self.addCleanup(shutil.rmtree, self.pack_dir)
self.pack_prefix = os.path.join(self.pack_dir, b'pack')
with open(self.pack_prefix + b'.pack', 'wb') as f:
build_pack(f, [
(REF_DELTA, (self.blobs[b'foo'].id, b'foo1234')),
(Blob.type_num, b'bar'),
(REF_DELTA, (self.blobs[b'bar'].id, b'bar2468'))],
store=self.store)
# Index the new pack.
with self.make_pack(True) as pack:
with PackData(pack._data_path) as data:
data.pack = pack
data.create_index(self.pack_prefix + b'.idx')
del self.store[self.blobs[b'bar'].id]
def setUp(self):
super(TestThinPack, self).setUp()
self.store = MemoryObjectStore()
self.blobs = {}
for blob in ('foo', 'bar', 'foo1234', 'bar2468'):
self.blobs[blob] = make_object(Blob, data=blob)
self.store.add_object(self.blobs['foo'])
self.store.add_object(self.blobs['bar'])
# Build a thin pack. 'foo' is as an external reference, 'bar' an
# internal reference.
self.pack_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.pack_dir)
self.pack_prefix = os.path.join(self.pack_dir, 'pack')
f = open(self.pack_prefix + '.pack', 'wb')
try:
build_pack(f, [
(REF_DELTA, (self.blobs['foo'].id, 'foo1234')),
(Blob.type_num, 'bar'),
(REF_DELTA, (self.blobs['bar'].id, 'bar2468'))],
store=self.store)
finally:
f.close()
# Index the new pack.
with self.make_pack(True) as pack:
with PackData(pack._data_path) as data:
data.pack = pack
data.create_index(self.pack_prefix + '.idx')
del self.store[self.blobs['bar'].id]
def move_in_pack(self, path):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
p = PackData(path)
try:
entries = p.sorted_entries()
basename = os.path.join(
self.pack_dir,
"pack-%s" % iter_sha1(entry[0] for entry in entries))
f = GitFile(basename + ".idx", "wb")
try:
write_pack_index_v2(f, entries, p.get_stored_checksum())
finally:
f.close()
finally:
p.close()
os.rename(path, basename + ".pack")
final_pack = Pack(basename)
self._add_known_pack(final_pack)
return final_pack
def move_in_pack(self, path):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
with PackData(path) as p:
entries = p.sorted_entries()
basename = self._get_pack_basepath(entries)
with GitFile(basename + ".idx", "wb") as f:
write_pack_index_v2(f, entries, p.get_stored_checksum())
if self._pack_cache is None or self._pack_cache_stale():
self._update_pack_cache()
try:
return self._pack_cache[basename]
except KeyError:
pass
else:
os.unlink(path)
try:
os.rename(path, basename + ".pack")
except OSError as e:
if e.errno == errno.EEXIST:
# This can happen on Windows..
# It's safe to ignore this, since if the file already exists,
# it should have the same contents as the one we just
# generated.
os.unlink(path)
else:
raise
final_pack = Pack(basename)
self._add_known_pack(basename, final_pack)
return final_pack
def commit():
if pf.tell() == 0:
pf.close()
return None
pf.seek(0)
p = PackData(pf.name, pf)
entries = p.sorted_entries()
basename = iter_sha1(entry[0] for entry in entries).decode('ascii')
idxf = tempfile.SpooledTemporaryFile()
checksum = p.get_stored_checksum()
write_pack_index_v2(idxf, entries, checksum)
idxf.seek(0)
idx = load_pack_index_file(basename + '.idx', idxf)
for pack in self.packs:
if pack.get_stored_checksum() == p.get_stored_checksum():
p.close()
idx.close()
return pack
pf.seek(0)
idxf.seek(0)
self._upload_pack(basename, pf, idxf)
final_pack = Pack.from_objects(p, idx)
self._add_cached_pack(basename, final_pack)
return final_pack
def setUp(self):
super(TestThinPack, self).setUp()
self.store = MemoryObjectStore()
self.blobs = {}
for blob in (b"foo", b"bar", b"foo1234", b"bar2468"):
self.blobs[blob] = make_object(Blob, data=blob)
self.store.add_object(self.blobs[b"foo"])
self.store.add_object(self.blobs[b"bar"])
# Build a thin pack. 'foo' is as an external reference, 'bar' an
# internal reference.
self.pack_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.pack_dir)
self.pack_prefix = os.path.join(self.pack_dir, "pack")
with open(self.pack_prefix + ".pack", "wb") as f:
build_pack(
f,
[
(REF_DELTA, (self.blobs[b"foo"].id, b"foo1234")),
(Blob.type_num, b"bar"),
(REF_DELTA, (self.blobs[b"bar"].id, b"bar2468")),
],
store=self.store,
)
# Index the new pack.
with self.make_pack(True) as pack:
with PackData(pack._data_path) as data:
data.pack = pack
data.create_index(self.pack_prefix + ".idx")
del self.store[self.blobs[b"bar"].id]
def move_in_pack(self, path):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
with PackData(path) as p:
entries = p.sorted_entries()
basename = self._get_pack_basepath(entries)
with GitFile(basename + ".idx", "wb") as f:
write_pack_index_v2(f, entries, p.get_stored_checksum())
if self._pack_cache is None or self._pack_cache_stale():
self._update_pack_cache()
try:
return self._pack_cache[basename]
except KeyError:
pass
else:
os.unlink(path)
os.rename(path, basename + ".pack")
final_pack = Pack(basename)
self._add_known_pack(basename, final_pack)
return final_pack
def move_in_pack(self, f):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
f.seek(0)
p = PackData("", f, len(f.getvalue()))
entries = p.sorted_entries()
basename = "pack-%s" % iter_sha1(entry[0]
for entry in entries).decode('ascii')
p._filename = basename + ".pack"
f.seek(0)
self.pack_transport.put_file(basename + ".pack", f)
idxfile = self.pack_transport.open_write_stream(basename + ".idx")
try:
write_pack_index_v2(idxfile, entries, p.get_stored_checksum())
finally:
idxfile.close()
idxfile = self.pack_transport.get(basename + ".idx")
idx = load_pack_index_file(basename + ".idx", idxfile)
final_pack = Pack.from_objects(p, idx)
final_pack._basename = basename
self._add_known_pack(basename, final_pack)
return final_pack
def move_in_thin_pack(self, path):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
data = PackData(path)
# Write index for the thin pack (do we really need this?)
temppath = os.path.join(self.pack_dir,
sha_to_hex(urllib2.randombytes(20))+".tempidx")
data.create_index_v2(temppath, self.get_raw)
p = Pack.from_objects(data, load_pack_index(temppath))
# Write a full pack version
temppath = os.path.join(self.pack_dir,
sha_to_hex(urllib2.randombytes(20))+".temppack")
write_pack(temppath, ((o, None) for o in p.iterobjects(self.get_raw)),
len(p))
pack_sha = load_pack_index(temppath+".idx").objects_sha1()
newbasename = os.path.join(self.pack_dir, "pack-%s" % pack_sha)
os.rename(temppath+".pack", newbasename+".pack")
os.rename(temppath+".idx", newbasename+".idx")
self._add_known_pack(newbasename)
def move_in_pack(self, path):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
with PackData(path) as p:
entries = p.sorted_entries()
basename = self._get_pack_basepath(entries)
with GitFile(basename + ".idx", "wb") as f:
write_pack_index_v2(f, entries, p.get_stored_checksum())
for pack in self.packs:
if pack._basename == basename:
return pack
target_pack = basename + '.pack'
if sys.platform == 'win32':
# Windows might have the target pack file lingering. Attempt
# removal, silently passing if the target does not exist.
try:
os.remove(target_pack)
except (IOError, OSError) as e:
if e.errno != errno.ENOENT:
raise
os.rename(path, target_pack)
final_pack = Pack(basename)
self._add_cached_pack(basename, final_pack)
return final_pack
def make_pack_iter(self, f, thin=None):
if thin is None:
thin = bool(list(self.store))
resolve_ext_ref = thin and self.get_raw_no_repeat or None
data = PackData('test.pack', file=f)
return TestPackIterator.for_pack_data(
data, resolve_ext_ref=resolve_ext_ref)
def _complete_thin_pack(self, f, path, copier, indexer):
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.flush()
# Move the pack in.
entries.sort()
pack_base_name = posixpath.join(
self.pack_dir,
"pack-" + os.fsdecode(iter_sha1(e[0] for e in entries)),
)
self.scon.put_object(pack_base_name + ".pack", f)
# Write the index.
filename = pack_base_name + ".idx"
index_file = BytesIO()
write_pack_index_v2(index_file, entries, pack_sha)
self.scon.put_object(filename, index_file)
# Write pack info.
f.seek(0)
pack_data = PackData(filename="", file=f)
index_file.seek(0)
pack_index = load_pack_index_file("", index_file)
serialized_pack_info = pack_info_create(pack_data, pack_index)
f.close()
index_file.close()
pack_info_file = BytesIO(serialized_pack_info)
filename = pack_base_name + ".info"
self.scon.put_object(filename, pack_info_file)
pack_info_file.close()
# Add the pack to the store and return it.
final_pack = SwiftPack(pack_base_name, scon=self.scon)
final_pack.check_length_and_checksum()
self._add_cached_pack(pack_base_name, final_pack)
return final_pack
def fetch_objects(self, determine_wants, graph_walker, progress=None):
fd, path = tempfile.mkstemp(suffix=".pack")
self.fetch_pack(determine_wants, graph_walker,
lambda x: os.write(fd, x), progress)
os.close(fd)
basename = path[:-len(".pack")]
p = PackData(path)
p.create_index_v2(basename + ".idx")
pack = Pack(basename)
os.remove(path)
return (len(p), pack.iterobjects())
def _load_packs(self):
ret = []
for name in self._pack_names():
if name.startswith("pack-") and name.endswith(".pack"):
try:
size = self.pack_transport.stat(name).st_size
except TransportNotPossible:
f = self.pack_transport.get(name)
pd = PackData(name, f)
else:
pd = PackData(name,
self.pack_transport.get(name),
size=size)
idxname = name.replace(".pack", ".idx")
idx = load_pack_index_file(idxname,
self.pack_transport.get(idxname))
pack = Pack.from_objects(pd, idx)
pack._basename = idxname[:-4]
ret.append(pack)
return ret
def test_checksum_mismatch(self):
with self.get_pack_data(pack1_sha) as data:
index = self.get_pack_index(pack1_sha)
Pack.from_objects(data, index).check_length_and_checksum()
data._file.seek(0)
bad_file = BytesIO(data._file.read()[:-20] + ('\xff' * 20))
bad_data = PackData('', file=bad_file)
bad_pack = Pack.from_lazy_objects(lambda: bad_data, lambda: index)
self.assertRaises(ChecksumMismatch, lambda: bad_pack.data)
self.assertRaises(ChecksumMismatch, lambda:
bad_pack.check_length_and_checksum())
def test_length_mismatch(self):
with self.get_pack_data(pack1_sha) as data:
index = self.get_pack_index(pack1_sha)
Pack.from_objects(data, index).check_length_and_checksum()
data._file.seek(12)
bad_file = BytesIO()
write_pack_header(bad_file, 9999)
bad_file.write(data._file.read())
bad_file = BytesIO(bad_file.getvalue())
bad_data = PackData('', file=bad_file)
bad_pack = Pack.from_lazy_objects(lambda: bad_data, lambda: index)
self.assertRaises(AssertionError, lambda: bad_pack.data)
self.assertRaises(AssertionError,
lambda: bad_pack.check_length_and_checksum())
def apply_pack(self, refs, read):
""" apply pack from client to current repository """
fd, path = tempfile.mkstemp(suffix=".pack")
f = os.fdopen(fd, 'w')
f.write(read())
f.close()
p = PackData(path)
entries = p.sorted_entries()
write_pack_index_v2(path[:-5] + ".idx", entries,
p.calculate_checksum())
def get_objects():
pack = Pack(path[:-5])
for obj in pack.iterobjects():
yield obj
target = Repository.open(self.directory)
target.lock_write()
try:
target.start_write_group()
try:
import_git_objects(target, self.mapping, iter(get_objects()))
finally:
target.commit_write_group()
finally:
target.unlock()
for oldsha, sha, ref in refs:
if ref[:11] == 'refs/heads/':
branch_nick = ref[11:]
try:
target_dir = BzrDir.open(self.directory + "/" +
branch_nick)
except:
target_dir = BzrDir.create(self.directory + "/" +
branch_nick)
try:
target_branch = target_dir.open_branch()
except:
target_branch = target_dir.create_branch()
rev_id = self.mapping.revision_id_foreign_to_bzr(sha)
target_branch.generate_revision_history(rev_id)
def move_in_pack(self, path):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
with PackData(path) as p:
entries = p.sorted_entries()
basename = self._get_pack_basepath(entries)
with GitFile(basename + b'.idx', "wb") as f:
write_pack_index_v2(f, entries, p.get_stored_checksum())
os.rename(path, basename + b'.pack')
final_pack = Pack(basename)
self._add_known_pack(basename, final_pack)
return final_pack
def commit():
f.seek(0)
pack = PackData(file=f, filename="")
entries = pack.sorted_entries()
if len(entries):
basename = posixpath.join(
self.pack_dir,
"pack-%s" % iter_sha1(entry[0] for entry in entries))
index = BytesIO()
write_pack_index_v2(index, entries, pack.get_stored_checksum())
self.scon.put_object(basename + ".pack", f)
f.close()
self.scon.put_object(basename + ".idx", index)
index.close()
final_pack = SwiftPack(basename, scon=self.scon)
final_pack.check_length_and_checksum()
self._add_known_pack(basename, final_pack)
return final_pack
else:
return None
def commit():
from .fetch import import_git_objects
os.fsync(fd)
f.close()
if os.path.getsize(path) == 0:
return
pd = PackData(path)
pd.create_index_v2(path[:-5] + ".idx", self.object_store.get_raw)
p = Pack(path[:-5])
with self.repository.lock_write():
self.repository.start_write_group()
try:
import_git_objects(self.repository, self.mapping,
p.iterobjects(get_raw=self.get_raw),
self.object_store)
except BaseException:
self.repository.abort_write_group()
raise
else:
self.repository.commit_write_group()
def get_pack_data(self, sha):
"""Returns a PackData object from the datadir with the given sha"""
return PackData(
os.path.join(self.datadir, "pack-%s.pack" % sha.decode("ascii")))
def _pack_data() -> PackData:
pack_data_bytes = self._http_get(f"objects/pack/{pack_name}")
return PackData(pack_name, file=pack_data_bytes)
def get_pack_data(self, sha):
"""Returns a PackData object from the datadir with the given sha"""
return PackData(os.path.join(self.datadir, 'pack-%s.pack' % sha))
