def get_packed_refs(self):
"""Get contents of the packed-refs file.
:return: Dictionary mapping ref names to SHA1s
:note: Will return an empty dictionary when no packed-refs file is
present.
"""
# TODO: invalidate the cache on repacking
if self._packed_refs is None:
# set both to empty because we want _peeled_refs to be
# None if and only if _packed_refs is also None.
self._packed_refs = {}
self._peeled_refs = {}
path = os.path.join(self.path, b'packed-refs')
try:
f = GitFile(path, 'rb')
except IOError as e:
if e.errno == errno.ENOENT:
return {}
raise
with f:
first_line = next(iter(f)).rstrip()
if (first_line.startswith(b'# pack-refs') and b' peeled' in
first_line):
for sha, name, peeled in read_packed_refs_with_peeled(f):
self._packed_refs[name] = sha
if peeled:
self._peeled_refs[name] = peeled
else:
f.seek(0)
for sha, name in read_packed_refs(f):
self._packed_refs[name] = sha
return self._packed_refs
def remove_if_equals(self, name, old_ref, committer=None, timestamp=None,
timezone=None, message=None):
"""Remove a refname only if it currently equals old_ref.
This method does not follow symbolic references. It can be used to
perform an atomic compare-and-delete operation.
:param name: The refname to delete.
:param old_ref: The old sha the refname must refer to, or None to
delete unconditionally.
:param message: Optional message
:return: True if the delete was successful, False otherwise.
"""
self._check_refname(name)
filename = self.refpath(name)
ensure_dir_exists(os.path.dirname(filename))
f = GitFile(filename, 'wb')
try:
if old_ref is not None:
orig_ref = self.read_loose_ref(name)
if orig_ref is None:
orig_ref = self.get_packed_refs().get(name, ZERO_SHA)
if orig_ref != old_ref:
return False
# remove the reference file itself
try:
os.remove(filename)
except OSError as e:
if e.errno != errno.ENOENT: # may only be packed
raise
self._remove_packed_ref(name)
self._log(name, old_ref, None, committer=committer,
timestamp=timestamp, timezone=timezone, message=message)
finally:
# never write, we just wanted the lock
f.abort()
# outside of the lock, clean-up any parent directory that might now
# be empty. this ensures that re-creating a reference of the same
# name of what was previously a directory works as expected
parent = name
while True:
try:
parent, _ = parent.rsplit(b'/', 1)
except ValueError:
break
parent_filename = self.refpath(parent)
try:
os.rmdir(parent_filename)
except OSError as e:
# this can be caused by the parent directory being
# removed by another process, being not empty, etc.
# in any case, this is non fatal because we already
# removed the reference, just ignore it
break
return True
def write(self):
"""Write current contents of index to disk."""
f = GitFile(self._filename, 'wb')
try:
f = SHA1Writer(f)
write_index_dict(f, self._byname)
finally:
f.close()
def test_abort_close_removed(self):
foo = self.path('foo')
f = GitFile(foo, 'wb')
f._file.close()
os.remove(foo + ".lock")
f.abort()
self.assertTrue(f._closed)
def test_readonly(self):
f = GitFile(self.path('foo'), 'rb')
self.assertTrue(isinstance(f, io.IOBase))
self.assertEqual(b'foo contents', f.read())
self.assertEqual(b'', f.read())
f.seek(4)
self.assertEqual(b'contents', f.read())
f.close()
def read_loose_ref(self, name):
"""Read a reference file and return its contents.
If the reference file a symbolic reference, only read the first line of
the file. Otherwise, only read the first 40 bytes.
:param name: the refname to read, relative to refpath
:return: The contents of the ref file, or None if the file does not
exist.
:raises IOError: if any other error occurs
"""
filename = self.refpath(name)
try:
with GitFile(filename, 'rb') as f:
header = f.read(len(SYMREF))
if header == SYMREF:
# Read only the first line
return header + next(iter(f)).rstrip(b'\r\n')
else:
# Read only the first 40 bytes
return header + f.read(40 - len(SYMREF))
except IOError as e:
if e.errno in (errno.ENOENT, errno.EISDIR):
return None
raise
def move_in_pack(self, path):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param path: Path to the pack file.
"""
with PackData(path) as p:
entries = p.sorted_entries()
basename = self._get_pack_basepath(entries)
with GitFile(basename + ".idx", "wb") as f:
write_pack_index_v2(f, entries, p.get_stored_checksum())
if self._pack_cache is None or self._pack_cache_stale():
self._update_pack_cache()
try:
return self._pack_cache[basename]
except KeyError:
pass
target_pack = basename + '.pack'
if sys.platform == 'win32':
# Windows might have the target pack file lingering. Attempt
# removal, silently passing if the target does not exist.
try:
os.remove(target_pack)
except (IOError, OSError) as e:
if e.errno != errno.ENOENT:
raise
os.rename(path, target_pack)
final_pack = Pack(basename)
self._add_known_pack(basename, final_pack)
return final_pack
def _read_alternate_paths(self):
try:
f = GitFile(os.path.join(self.path, INFODIR, "alternates"), 'rb')
except (OSError, IOError) as e:
if e.errno == errno.ENOENT:
return
raise
with f:
for line in f.readlines():
line = line.rstrip(b"\n")
if line[0] == b"#":
continue
if os.path.isabs(line):
yield line.decode(sys.getfilesystemencoding())
else:
yield os.path.join(self.path, line).decode(
sys.getfilesystemencoding())
def read(self):
"""Read current contents of index from disk."""
if not os.path.exists(self._filename):
return
f = GitFile(self._filename, 'rb')
try:
f = SHA1Reader(f)
for x in read_index(f):
self[x[0]] = IndexEntry(*x[1:])
# FIXME: Additional data?
f.read(os.path.getsize(self._filename) - f.tell() - 20)
f.check_sha()
finally:
f.close()
def stashes(self):
reflog_path = os.path.join(self._repo.commondir(), 'logs', self._ref)
try:
with GitFile(reflog_path, 'rb') as f:
return reversed(list(read_reflog(f)))
except EnvironmentError as e:
if e.errno == errno.ENOENT:
return []
raise
def test_write(self):
foo = self.path('foo')
foo_lock = '%s.lock' % foo
orig_f = open(foo, 'rb')
self.assertEqual(orig_f.read(), b'foo contents')
orig_f.close()
self.assertFalse(os.path.exists(foo_lock))
f = GitFile(foo, 'wb')
self.assertFalse(f.closed)
self.assertRaises(AttributeError, getattr, f, 'not_a_file_property')
self.assertTrue(os.path.exists(foo_lock))
f.write(b'new stuff')
f.seek(4)
f.write(b'contents')
f.close()
self.assertFalse(os.path.exists(foo_lock))
new_f = open(foo, 'rb')
self.assertEqual(b'new contents', new_f.read())
new_f.close()
def add_object(self, obj):
"""Add a single object to this object store.
:param obj: Object to add
"""
path = self._get_shafile_path(obj.id)
dir = os.path.dirname(path)
try:
os.mkdir(dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if os.path.exists(path):
return # Already there, no need to write again
with GitFile(path, 'wb') as f:
f.write(obj.as_legacy_object())
def set_if_equals(self, name, old_ref, new_ref, committer=None,
timestamp=None, timezone=None, message=None):
"""Set a refname to new_ref only if it currently equals old_ref.
This method follows all symbolic references, and can be used to perform
an atomic compare-and-swap operation.
:param name: The refname to set.
:param old_ref: The old sha the refname must refer to, or None to set
unconditionally.
:param new_ref: The new sha the refname will refer to.
:param message: Set message for reflog
:return: True if the set was successful, False otherwise.
"""
self._check_refname(name)
try:
realnames, _ = self.follow(name)
realname = realnames[-1]
except (KeyError, IndexError):
realname = name
filename = self.refpath(realname)
ensure_dir_exists(os.path.dirname(filename))
with GitFile(filename, 'wb') as f:
if old_ref is not None:
try:
# read again while holding the lock
orig_ref = self.read_loose_ref(realname)
if orig_ref is None:
orig_ref = self.get_packed_refs().get(
realname, ZERO_SHA)
if orig_ref != old_ref:
f.abort()
return False
except (OSError, IOError):
f.abort()
raise
try:
f.write(new_ref + b'\n')
except (OSError, IOError):
f.abort()
raise
self._log(realname, old_ref, new_ref, committer=committer,
timestamp=timestamp, timezone=timezone, message=message)
return True
def set_symbolic_ref(self, name, other, committer=None, timestamp=None,
timezone=None, message=None):
"""Make a ref point at another ref.
:param name: Name of the ref to set
:param other: Name of the ref to point at
:param message: Optional message to describe the change
"""
self._check_refname(name)
self._check_refname(other)
filename = self.refpath(name)
f = GitFile(filename, 'wb')
try:
f.write(SYMREF + other + b'\n')
sha = self.follow(name)[-1]
self._log(name, sha, sha, committer=committer,
timestamp=timestamp, timezone=timezone,
message=message)
except BaseException:
f.abort()
raise
else:
f.close()
def test_abort(self):
foo = self.path('foo')
foo_lock = '%s.lock' % foo
orig_f = open(foo, 'rb')
self.assertEqual(orig_f.read(), b'foo contents')
orig_f.close()
f = GitFile(foo, 'wb')
f.write(b'new contents')
f.abort()
self.assertTrue(f.closed)
self.assertFalse(os.path.exists(foo_lock))
new_orig_f = open(foo, 'rb')
self.assertEqual(new_orig_f.read(), b'foo contents')
new_orig_f.close()
def add_alternate_path(self, path):
"""Add an alternate path to this object store.
"""
try:
os.mkdir(os.path.join(self.path, INFODIR))
except OSError as e:
if e.errno != errno.EEXIST:
raise
alternates_path = os.path.join(self.path, INFODIR, "alternates")
with GitFile(alternates_path, 'wb') as f:
try:
orig_f = open(alternates_path, 'rb')
except (OSError, IOError) as e:
if e.errno != errno.ENOENT:
raise
else:
with orig_f:
f.write(orig_f.read())
f.write(path.encode(sys.getfilesystemencoding()) + b"\n")
if not os.path.isabs(path):
path = os.path.join(self.path, path)
self.alternates.append(DiskObjectStore(path))
def add_if_new(self, name, ref, committer=None, timestamp=None,
timezone=None, message=None):
"""Add a new reference only if it does not already exist.
This method follows symrefs, and only ensures that the last ref in the
chain does not exist.
:param name: The refname to set.
:param ref: The new sha the refname will refer to.
:param message: Optional message for reflog
:return: True if the add was successful, False otherwise.
"""
try:
realnames, contents = self.follow(name)
if contents is not None:
return False
realname = realnames[-1]
except (KeyError, IndexError):
realname = name
self._check_refname(realname)
filename = self.refpath(realname)
ensure_dir_exists(os.path.dirname(filename))
with GitFile(filename, 'wb') as f:
if os.path.exists(filename) or name in self.get_packed_refs():
f.abort()
return False
try:
f.write(ref + b'\n')
except (OSError, IOError):
f.abort()
raise
else:
self._log(name, None, ref, committer=committer,
timestamp=timestamp, timezone=timezone,
message=message)
return True
def _remove_packed_ref(self, name):
if self._packed_refs is None:
return
filename = os.path.join(self.path, b'packed-refs')
# reread cached refs from disk, while holding the lock
f = GitFile(filename, 'wb')
try:
self._packed_refs = None
self.get_packed_refs()
if name not in self._packed_refs:
return
del self._packed_refs[name]
if name in self._peeled_refs:
del self._peeled_refs[name]
write_packed_refs(f, self._packed_refs, self._peeled_refs)
f.close()
finally:
f.abort()
def from_path(cls, path):
"""Open a SHA file from disk."""
with GitFile(path, 'rb') as f:
return cls.from_file(f)
def write_to_path(self, path=None):
"""Write configuration to a file on disk."""
if path is None:
path = self.path
with GitFile(path, 'wb') as f:
self.write_to_file(f)
def test_default_mode(self):
f = GitFile(self.path('foo'))
self.assertEqual(b'foo contents', f.read())
f.close()
def test_open_twice(self):
foo = self.path('foo')
f1 = GitFile(foo, 'wb')
f1.write(b'new')
try:
f2 = GitFile(foo, 'wb')
self.fail()
except FileLocked:
pass
else:
f2.close()
f1.write(b' contents')
f1.close()
# Ensure trying to open twice doesn't affect original.
f = open(foo, 'rb')
self.assertEqual(b'new contents', f.read())
f.close()
def test_abort_close(self):
foo = self.path('foo')
f = GitFile(foo, 'wb')
f.abort()
try:
f.close()
except (IOError, OSError):
self.fail()
f = GitFile(foo, 'wb')
f.close()
try:
f.abort()
except (IOError, OSError):
self.fail()
def from_path(cls, path):
"""Read configuration from a file on disk."""
with GitFile(path, 'rb') as f:
ret = cls.from_file(f)
ret.path = path
return ret
def _complete_thin_pack(self, f, path, copier, indexer):
"""Move a specific file containing a pack into the pack directory.
:note: The file should be on the same file system as the
packs directory.
:param f: Open file object for the pack.
:param path: Path to the pack file.
:param copier: A PackStreamCopier to use for writing pack data.
:param indexer: A PackIndexer for indexing the pack.
"""
entries = list(indexer)
# Update the header with the new number of objects.
f.seek(0)
write_pack_header(f, len(entries) + len(indexer.ext_refs()))
# Must flush before reading (http://bugs.python.org/issue3207)
f.flush()
# Rescan the rest of the pack, computing the SHA with the new header.
new_sha = compute_file_sha(f, end_ofs=-20)
# Must reposition before writing (http://bugs.python.org/issue3207)
f.seek(0, os.SEEK_CUR)
# Complete the pack.
for ext_sha in indexer.ext_refs():
assert len(ext_sha) == 20
type_num, data = self.get_raw(ext_sha)
offset = f.tell()
crc32 = write_pack_object(f, type_num, data, sha=new_sha)
entries.append((ext_sha, offset, crc32))
pack_sha = new_sha.digest()
f.write(pack_sha)
f.close()
# Move the pack in.
entries.sort()
pack_base_name = self._get_pack_basepath(entries)
target_pack = pack_base_name + '.pack'
if sys.platform == 'win32':
# Windows might have the target pack file lingering. Attempt
# removal, silently passing if the target does not exist.
try:
os.remove(target_pack)
except (IOError, OSError) as e:
if e.errno != errno.ENOENT:
raise
os.rename(path, target_pack)
# Write the index.
index_file = GitFile(pack_base_name + '.idx', 'wb')
try:
write_pack_index_v2(index_file, entries, pack_sha)
index_file.close()
finally:
index_file.abort()
# Add the pack to the store and return it.
final_pack = Pack(pack_base_name)
final_pack.check_length_and_checksum()
self._add_known_pack(pack_base_name, final_pack)
return final_pack
def test_remove_packed_without_peeled(self):
refs_file = os.path.join(self._repo.path, 'packed-refs')
f = GitFile(refs_file)
refs_data = f.read()
f.close()
f = GitFile(refs_file, 'wb')
f.write(b'\n'.join(l for l in refs_data.split(b'\n')
if not l or l[0] not in b'#^'))
f.close()
self._repo = Repo(self._repo.path)
refs = self._repo.refs
self.assertTrue(
refs.remove_if_equals(b'refs/heads/packed',
b'42d06bd4b77fed026b154d16493e5deab78f02ec'))
def writeIndex(self, filename, entries, pack_checksum):
# FIXME: Write to BytesIO instead rather than hitting disk ?
with GitFile(filename, "wb") as f:
self._write_fn(f, entries, pack_checksum)