def test_commit_gpgsig_parsing():
c = git.parse_commit(gpgsig_example_1)
assert c.gpgsig
assert c.gpgsig.startswith(b'-----BEGIN PGP SIGNATURE-----\n')
assert c.gpgsig.endswith(b'\n-----END PGP SIGNATURE-----\n')
c = git.parse_commit(gpgsig_example_2)
assert c.gpgsig
assert c.gpgsig.startswith(b'-----BEGIN PGP SIGNATURE-----')
assert c.gpgsig.endswith(b'\n-----END PGP SIGNATURE-----\n\n')
def root_items(repo, names=None, want_meta=True):
"""Yield (name, item) for the items in '/' in the VFS. Return
everything if names is logically false, otherwise return only
items with a name in the collection.
"""
# FIXME: what about non-leaf refs like 'refs/heads/foo/bar/baz?
global _root, _tags
if not names:
yield '.', _root
yield '.tag', _tags
# FIXME: maybe eventually support repo.clone() or something
# and pass in two repos, so we can drop the tuple() and stream
# in parallel (i.e. meta vs refs).
for name, oid in tuple(repo.refs([], limit_to_heads=True)):
assert(name.startswith('refs/heads/'))
yield name[11:], _revlist_item_from_oid(repo, oid, want_meta)
return
if '.' in names:
yield '.', _root
if '.tag' in names:
yield '.tag', _tags
for ref in names:
if ref in ('.', '.tag'):
continue
it = repo.cat('refs/heads/' + ref)
oidx, typ, size = next(it)
if not oidx:
for _ in it: pass
continue
assert typ == 'commit'
commit = parse_commit(''.join(it))
yield ref, _revlist_item_from_oid(repo, oidx.decode('hex'), want_meta)
def rev_list(self, ref_or_refs, count=None, parse=None, format=None):
# TODO: maybe we should refactor this to not have all of bup rely
# on the git format ... it's ugly that we have to produce it here
#
# TODO: also, this is weird, I'm using existing bup functionality
# to pretend I'm git, and then bup uses that again, really it stands
# to reason that bup should do this itself without even *having* a
# rev_list() method that calls out to git - and it'll probably be
# faster too since we have the bloom/midx.
assert count is None
assert format in (b'%T %at', None)
# TODO: ugh, this is a messy API ...
if isinstance(ref_or_refs, compat.str_type):
ref = ref_or_refs
else:
assert len(ref_or_refs) == 1
ref = ref_or_refs[0]
while True:
commit = git.parse_commit(
git.get_cat_data(self.cat(ref), b'commit'))
if format is None:
yield ref
else:
if format == b'%T %at':
data = BytesIO(b'%s %d\n' %
(commit.tree, commit.author_sec))
yield (ref, parse(data))
if not commit.parents:
break
ref = commit.parents[0]
def root_items(repo, names=None, want_meta=True):
"""Yield (name, item) for the items in '/' in the VFS. Return
everything if names is logically false, otherwise return only
items with a name in the collection.
"""
# FIXME: what about non-leaf refs like 'refs/heads/foo/bar/baz?
global _root, _tags
if not names:
yield '.', _root
yield '.tag', _tags
# FIXME: maybe eventually support repo.clone() or something
# and pass in two repos, so we can drop the tuple() and stream
# in parallel (i.e. meta vs refs).
for name, oid in tuple(repo.refs([], limit_to_heads=True)):
assert(name.startswith('refs/heads/'))
yield name[11:], _revlist_item_from_oid(repo, oid, want_meta)
return
if '.' in names:
yield '.', _root
if '.tag' in names:
yield '.tag', _tags
for ref in names:
if ref in ('.', '.tag'):
continue
it = repo.cat('refs/heads/' + ref)
oidx, typ, size = next(it)
if not oidx:
for _ in it: pass
continue
assert typ == 'commit'
commit = parse_commit(''.join(it))
yield ref, _revlist_item_from_oid(repo, oidx.decode('hex'), want_meta)
def contents(repo, item, names=None, want_meta=True):
"""Yields information about the items contained in item. Yields
(name, item) for each name in names, if the name exists, in an
unspecified order. If there are no names, then yields (name,
item) for all items, including, a first item named '.'
representing the container itself.
Any given name might produce more than one result. For example,
saves to a branch that happen within the same second currently end
up with the same VFS timestmap, i.e. /foo/2017-09-10-150833/.
Note that want_meta is advisory. For any given item, item.meta
might be a Metadata instance or a mode, and if the former,
meta.size might be None. Missing sizes can be computed via via
item_size() or augment_item_meta(..., include_size=True).
Do not modify any item.meta Metadata instances directly. If
needed, make a copy via item.meta.copy() and modify that instead.
"""
# Q: are we comfortable promising '.' first when no names?
assert repo
assert S_ISDIR(item_mode(item))
item_t = type(item)
if item_t == Item:
it = repo.cat(item.oid.encode('hex'))
_, obj_type, size = next(it)
data = ''.join(it)
if obj_type == 'tree':
if want_meta:
item_gen = tree_items_with_meta(repo, item.oid, data, names)
else:
item_gen = tree_items(item.oid, data, names)
elif obj_type == 'commit':
tree_oidx = parse_commit(data).tree
it = repo.cat(tree_oidx)
_, obj_type, size = next(it)
assert obj_type == 'tree'
tree_data = ''.join(it)
if want_meta:
item_gen = tree_items_with_meta(repo, tree_oidx.decode('hex'),
tree_data, names)
else:
item_gen = tree_items(tree_oidx.decode('hex'), tree_data,
names)
else:
for _ in it:
pass
raise Exception('unexpected git ' + obj_type)
elif item_t == RevList:
item_gen = revlist_items(repo, item.oid, names)
elif item_t == Root:
item_gen = root_items(repo, names)
elif item_t == Tags:
item_gen = tags_items(repo, names)
else:
raise Exception('unexpected VFS item ' + str(item))
for x in item_gen:
yield x
def handle_replace(item, src_repo, writer, opt):
assert(item.spec.method == 'replace')
if item.dest.path.startswith(b'/.tag/'):
get_random_item(item.spec.src, hexlify(item.src.hash),
src_repo, writer, opt)
return (item.src.hash,)
assert(item.dest.type == 'branch' or not item.dest.type)
src_oidx = hexlify(item.src.hash)
get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
commit_items = parse_commit(get_cat_data(src_repo.cat(src_oidx), b'commit'))
return item.src.hash, unhexlify(commit_items.tree)
def handle_replace(item, src_repo, writer, opt):
assert (item.spec.method == 'replace')
if item.dest.path.startswith('/.tag/'):
get_random_item(item.spec.src, item.src.hash.encode('hex'), src_repo,
writer, opt)
return (item.src.hash, )
assert (item.dest.type == 'branch' or not item.dest.type)
src_oidx = item.src.hash.encode('hex')
get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
commit_items = parse_commit(get_cat_data(src_repo.cat(src_oidx), 'commit'))
return item.src.hash, commit_items.tree.decode('hex')
def handle_ff(item, src_repo, writer, opt):
assert item.spec.method == 'ff'
assert item.src.type in ('branch', 'save', 'commit')
src_oidx = item.src.hash.encode('hex')
dest_oidx = item.dest.hash.encode('hex') if item.dest.hash else None
if not dest_oidx or dest_oidx in src_repo.rev_list(src_oidx):
# Can fast forward.
get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
commit_items = parse_commit(get_cat_data(src_repo.cat(src_oidx), 'commit'))
return item.src.hash, commit_items.tree.decode('hex')
spec_args = '%s %s' % (item.spec.argopt, item.spec.argval)
misuse('destination is not an ancestor of source for %r' % spec_args)
def handle_ff(item, src_repo, writer, opt):
assert item.spec.method == 'ff'
assert item.src.type in ('branch', 'save', 'commit')
src_oidx = hexlify(item.src.hash)
dest_oidx = hexlify(item.dest.hash) if item.dest.hash else None
if not dest_oidx or dest_oidx in src_repo.rev_list(src_oidx):
# Can fast forward.
get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
commit_items = parse_commit(get_cat_data(src_repo.cat(src_oidx), b'commit'))
return item.src.hash, unhexlify(commit_items.tree)
misuse('destination is not an ancestor of source for %s'
% spec_msg(item.spec))
def append_commit(name, hash, parent, src_repo, dest_repo, opt):
now = time.time()
items = parse_commit(get_cat_data(src_repo.cat(hash), b'commit'))
tree = unhexlify(items.tree)
author = b'%s <%s>' % (items.author_name, items.author_mail)
author_time = (items.author_sec, items.author_offset)
committer = b'%s <%s@%s>' % (userfullname(), username(), hostname())
get_random_item(name, hexlify(tree), src_repo, dest_repo, opt)
c = dest_repo.write_commit(tree, parent, author, items.author_sec,
items.author_offset, committer, now, None,
items.message)
return c, tree
def append_commit(name, hash, parent, src_repo, writer, opt):
now = time.time()
items = parse_commit(get_cat_data(src_repo.cat(hash), 'commit'))
tree = items.tree.decode('hex')
author = '%s <%s>' % (items.author_name, items.author_mail)
author_time = (items.author_sec, items.author_offset)
committer = '%s <%s@%s>' % (userfullname(), username(), hostname())
get_random_item(name, tree.encode('hex'), src_repo, writer, opt)
c = writer.new_commit(tree, parent, author, items.author_sec,
items.author_offset, committer, now, None,
items.message)
return c, tree
def tag_item(oid):
assert len(oid) == 20
oidx = oid.encode('hex')
it = repo.cat(oidx)
_, typ, size = next(it)
if typ == 'commit':
tree_oid = parse_commit(''.join(it)).tree.decode('hex')
assert len(tree_oid) == 20
# FIXME: more efficient/bulk?
return RevList(meta=_commit_meta_from_oidx(repo, oidx), oid=oid)
for _ in it:
pass
if typ == 'blob':
return Item(meta=default_file_mode, oid=oid)
elif typ == 'tree':
return Item(meta=default_dir_mode, oid=oid)
raise Exception('unexpected tag type ' + typ + ' for tag ' + name)
def tree_data_and_bupm(repo, oid):
"""Return (tree_bytes, bupm_oid) where bupm_oid will be None if the
tree has no metadata (i.e. older bup save, or non-bup tree).
"""
assert len(oid) == 20
it = repo.cat(hexlify(oid))
_, item_t, size = next(it)
data = b''.join(it)
if item_t == b'commit':
commit = parse_commit(data)
it = repo.cat(commit.tree)
_, item_t, size = next(it)
data = b''.join(it)
assert item_t == b'tree'
elif item_t != b'tree':
raise Exception('%s is not a tree or commit' % hexstr(oid))
for _, mangled_name, sub_oid in tree_decode(data):
if mangled_name == b'.bupm':
return data, sub_oid
if mangled_name > b'.bupm':
break
return data, None
def tree_data_and_bupm(repo, oid):
"""Return (tree_bytes, bupm_oid) where bupm_oid will be None if the
tree has no metadata (i.e. older bup save, or non-bup tree).
"""
assert len(oid) == 20
it = repo.cat(oid.encode('hex'))
_, item_t, size = next(it)
data = ''.join(it)
if item_t == 'commit':
commit = parse_commit(data)
it = repo.cat(commit.tree)
_, item_t, size = next(it)
data = ''.join(it)
assert item_t == 'tree'
elif item_t != 'tree':
raise Exception('%r is not a tree or commit' % oid.encode('hex'))
for _, mangled_name, sub_oid in tree_decode(data):
if mangled_name == '.bupm':
return data, sub_oid
if mangled_name > '.bupm':
break
return data, None
def get_commit_items(repo, hash):
data = git.get_cat_data(repo.cat(hash), b'commit')
return git.parse_commit(data)
def _commit_item_from_data(oid, data):
info = parse_commit(data)
return Commit(meta=default_dir_mode, oid=unhexlify(info.tree), coid=oid)
def _commit_item_from_data(oid, data):
info = parse_commit(data)
return Commit(meta=default_dir_mode,
oid=info.tree.decode('hex'),
coid=oid)
def cache_commit(repo, oid, require_meta=True, names=None):
"""Build, cache, and return a "name -> commit_item" dict of the entire
commit rev-list.
"""
entries = {}
entries[b'.'] = _revlist_item_from_oid(repo, oid, require_meta)
# Check if this a query related to a /latest/ query, i.e. if it
# is cached in the latest cache for this oid and this path, then
# we've been asked for <oid>/latest/ before and cached it, and
# then we can resolve it easily...
# This is necessary so that doing a <oid>/latest/ with the link
# we create below doesn't have to then go back to a full rev-list
# to resolve the symlink, and if it does it may even fail due to
# the missing _reverse_suffix_duplicates() in the /latest/ code.
if names and len(names) == 2 and names[0] == b'.':
latest_key = b'lat:' + oid
ret = cache_get(latest_key)
if ret and names[1] in ret:
return ret
# Special-case <oid>/latest/ queries so that we can do this faster
# than doing a whole rev-list, which can take a very long time.
if names in ((b'latest', ), (b'.', b'latest')):
latest_key = b'lat:' + oid
oidx = hexlify(oid)
it = repo.cat(oidx)
_, _, _ = next(it)
info = git.parse_commit(b''.join(it))
latest_rev = oidx, (unhexlify(info.tree), info.author_sec)
# This will not append the suffix as _reverse_suffix_duplicates()
# would, but it shouldn't really matter since we have no suffix
# here, but if you _do_ end up with one, the non-suffix version
# won't exist. So even if you 'remember' from a /latest/ run then
# you simply can't port it over to a clashing run with all.
latest_name = _name_for_rev(latest_rev)
latest_item = _item_for_rev(latest_rev)
ret = {
b'.': None, # HACK for ls -s, not sure why we're even
# asked for this entry? nobody cares if
# we already gave <branch>/latest/
b'latest': FakeLink(meta=default_symlink_mode, target=latest_name),
latest_name: latest_item,
}
cache_notice(latest_key, ret)
return ret
revs = repo.rev_list((hexlify(oid), ), format=b'%T %at', parse=parse_rev)
rev_items, rev_names = tee(revs)
revs = None # Don't disturb the tees
rev_names = _reverse_suffix_duplicates(_name_for_rev(x) for x in rev_names)
rev_items = (_item_for_rev(x) for x in rev_items)
tip = None
for item in rev_items:
name = next(rev_names)
tip = tip or (name, item)
entries[name] = item
entries[b'latest'] = FakeLink(meta=default_symlink_mode, target=tip[0])
revlist_key = b'rvl:' + tip[1].coid
entries[_HAS_META_ENTRY] = require_meta
cache_notice(revlist_key, entries, overwrite=True)
return entries
def _commit_meta_from_oidx(repo, oidx):
it = repo.cat(oidx)
_, typ, size = next(it)
assert typ == 'commit'
author_sec = parse_commit(''.join(it)).author_sec
return _commit_meta_from_auth_sec(author_sec)
def _commit_item_from_data(oid, data):
info = parse_commit(data)
return Commit(meta=default_dir_mode,
oid=info.tree.decode('hex'),
coid=oid)
def get_commit_items(id, cp):
return git.parse_commit(git.get_cat_data(cp.get(id), b'commit'))