def _mksubs(self):
self._subs = {}
it = cp(self._repo_dir).get(self.hash.encode('hex'))
type = it.next()
if type == 'commit':
del it
it = cp(self._repo_dir).get(self.hash.encode('hex') + ':')
type = it.next()
assert(type == 'tree')
for (mode,mangled_name,sha) in git.tree_decode(''.join(it)):
if mangled_name == '.bupm':
bupmode = stat.S_ISDIR(mode) and BUP_CHUNKED or BUP_NORMAL
self._bupm = File(self, mangled_name, GIT_MODE_FILE, sha,
bupmode)
continue
name, bupmode = git.demangle_name(mangled_name, mode)
if bupmode == git.BUP_CHUNKED:
mode = GIT_MODE_FILE
if stat.S_ISDIR(mode):
self._subs[name] = Dir(self, name, mode, sha, self._repo_dir)
elif stat.S_ISLNK(mode):
self._subs[name] = Symlink(self, name, sha, bupmode,
self._repo_dir)
else:
self._subs[name] = File(self, name, mode, sha, bupmode,
self._repo_dir)
def __init__(self, repo, oid, startofs):
it = repo.cat(oid.encode('hex'))
_, obj_t, size = next(it)
isdir = obj_t == 'tree'
data = ''.join(it)
if isdir:
self.it = _tree_chunks(repo, tree_decode(data), startofs)
self.blob = None
else:
self.it = None
self.blob = data[startofs:]
self.ofs = startofs
def _normal_or_chunked_file_size(repo, oid):
"""Return the size of the normal or chunked file indicated by oid."""
# FIXME: --batch-format CatPipe?
it = repo.cat(oid.encode('hex'))
_, obj_t, size = next(it)
ofs = 0
while obj_t == 'tree':
mode, name, last_oid = last(tree_decode(''.join(it)))
ofs += int(name, 16)
it = repo.cat(last_oid.encode('hex'))
_, obj_t, size = next(it)
return ofs + sum(len(b) for b in it)
def tree_items_with_meta(repo, oid, tree_data, names):
# For now, the .bupm order doesn't quite match git's, and we don't
# load the tree data incrementally anyway, so we just work in RAM
# via tree_data.
assert len(oid) == 20
bupm = None
for _, mangled_name, sub_oid in tree_decode(tree_data):
if mangled_name == '.bupm':
bupm = _FileReader(repo, sub_oid)
break
if mangled_name > '.bupm':
break
for item in tree_items(oid, tree_data, names, bupm):
yield item
def walk_object(cat_pipe, id, verbose=None, parent_path=[], writer=None):
# Yield everything reachable from id via cat_pipe, stopping
# whenever we hit something writer already has. Produce (id, type
# data) for each item. Since maybe_write() can't accept an
# iterator, join()ing the data here doesn't hurt anything.
item_it = cat_pipe.get(id)
type = item_it.next()
data = ''.join(item_it)
id = git.calc_hash(type, data)
if writer and writer.exists(id):
return
if type == 'blob':
yield (id, type, data)
elif type == 'commit':
yield (id, type, data)
commit_items = parse_commit(data)
tree_id = commit_items.tree
for x in walk_object(cat_pipe, tree_id, verbose, parent_path, writer):
yield x
parents = commit_items.parents
for pid in parents:
for x in walk_object(cat_pipe, pid, verbose, parent_path, writer):
yield x
elif type == 'tree':
yield (id, type, data)
for (mode, name, ent_id) in git.tree_decode(data):
if not verbose > 1:
for x in walk_object(cat_pipe, ent_id.encode('hex'),
writer=writer):
yield x
else:
demangled, bup_type = git.demangle_name(name)
sub_path = parent_path + [demangled]
# Don't print the sub-parts of chunked files.
sub_v = verbose if bup_type == git.BUP_NORMAL else None
for x in walk_object(cat_pipe, ent_id.encode('hex'),
sub_v, sub_path, writer):
yield x
if stat.S_ISDIR(mode):
if verbose > 1 and bup_type == git.BUP_NORMAL:
log('%s/\n' % '/'.join(sub_path))
elif verbose > 2: # (and BUP_CHUNKED)
log('%s\n' % '/'.join(sub_path))
elif verbose > 2:
log('%s\n' % '/'.join(sub_path))
else:
raise Exception('unexpected repository object type %r' % type)
def _tree_chunks(repo, tree, startofs):
"Tree should be a sequence of (name, mode, hash) as per tree_decode()."
assert(startofs >= 0)
# name is the chunk's hex offset in the original file
for mode, name, oid in _skip_chunks_before_offset(tree, startofs):
ofs = int(name, 16)
skipmore = startofs - ofs
if skipmore < 0:
skipmore = 0
it = repo.cat(oid.encode('hex'))
_, obj_t, size = next(it)
data = ''.join(it)
if S_ISDIR(mode):
assert obj_t == 'tree'
for b in _tree_chunks(repo, tree_decode(data), skipmore):
yield b
else:
assert obj_t == 'blob'
yield data[skipmore:]
def _tree_chunks(repo, tree, startofs):
"Tree should be a sequence of (name, mode, hash) as per tree_decode()."
assert (startofs >= 0)
# name is the chunk's hex offset in the original file
for mode, name, oid in _skip_chunks_before_offset(tree, startofs):
ofs = int(name, 16)
skipmore = startofs - ofs
if skipmore < 0:
skipmore = 0
it = repo.cat(hexlify(oid))
_, obj_t, size = next(it)
data = b''.join(it)
if S_ISDIR(mode):
assert obj_t == b'tree'
for b in _tree_chunks(repo, tree_decode(data), skipmore):
yield b
else:
assert obj_t == b'blob'
yield data[skipmore:]
def _mksubs(self):
self._subs = {}
it = cp().get(self.hash.encode('hex'))
type = it.next()
if type == 'commit':
del it
it = cp().get(self.hash.encode('hex') + ':')
type = it.next()
assert(type == 'tree')
for (mode,mangled_name,sha) in git.tree_decode(''.join(it)):
name = mangled_name
(name,bupmode) = git.demangle_name(mangled_name)
if bupmode == git.BUP_CHUNKED:
mode = GIT_MODE_FILE
if stat.S_ISDIR(mode):
self._subs[name] = Dir(self, name, mode, sha)
elif stat.S_ISLNK(mode):
self._subs[name] = Symlink(self, name, sha, bupmode)
else:
self._subs[name] = File(self, name, mode, sha, bupmode)
def _tree_chunks(repo, tree, startofs):
"Tree should be a sequence of (name, mode, hash) as per tree_decode()."
assert(startofs >= 0)
# name is the chunk's hex offset in the original file
tree = dropwhile(lambda (_1, name, _2): int(name, 16) < startofs, tree)
for mode, name, oid in tree:
ofs = int(name, 16)
skipmore = startofs - ofs
if skipmore < 0:
skipmore = 0
it = repo.cat(oid.encode('hex'))
_, obj_t, size = next(it)
data = ''.join(it)
if S_ISDIR(mode):
assert obj_t == 'tree'
for b in _tree_chunks(repo, tree_decode(data), skipmore):
yield b
else:
assert obj_t == 'blob'
yield data[skipmore:]
def ordered_tree_entries(tree_data, bupm=None):
"""Yields (name, mangled_name, kind, gitmode, oid) for each item in
tree, sorted by name.
"""
# Sadly, the .bupm entries currently aren't in git tree order,
# i.e. they don't account for the fact that git sorts trees
# (including our chunked trees) as if their names ended with "/",
# so "fo" sorts after "fo." iff fo is a directory. This makes
# streaming impossible when we need the metadata.
def result_from_tree_entry(tree_entry):
gitmode, mangled_name, oid = tree_entry
name, kind = git.demangle_name(mangled_name, gitmode)
return name, mangled_name, kind, gitmode, oid
tree_ents = (result_from_tree_entry(x) for x in tree_decode(tree_data))
if bupm:
tree_ents = sorted(tree_ents, key=lambda x: x[0])
for ent in tree_ents:
yield ent
def _mksubs(self):
self._subs = {}
it = cp().get(self.hash.encode('hex'))
type = it.next()
if type == 'commit':
del it
it = cp().get(self.hash.encode('hex') + ':')
type = it.next()
assert (type == 'tree')
for (mode, mangled_name, sha) in git.tree_decode(''.join(it)):
name = mangled_name
(name, bupmode) = git.demangle_name(mangled_name)
if bupmode == git.BUP_CHUNKED:
mode = GIT_MODE_FILE
if stat.S_ISDIR(mode):
self._subs[name] = Dir(self, name, mode, sha)
elif stat.S_ISLNK(mode):
self._subs[name] = Symlink(self, name, sha, bupmode)
else:
self._subs[name] = File(self, name, mode, sha, bupmode)
def ordered_tree_entries(tree_data, bupm=None):
"""Yields (name, mangled_name, kind, gitmode, oid) for each item in
tree, sorted by name.
"""
# Sadly, the .bupm entries currently aren't in git tree order,
# i.e. they don't account for the fact that git sorts trees
# (including our chunked trees) as if their names ended with "/",
# so "fo" sorts after "fo." iff fo is a directory. This makes
# streaming impossible when we need the metadata.
def result_from_tree_entry(tree_entry):
gitmode, mangled_name, oid = tree_entry
name, kind = git.demangle_name(mangled_name, gitmode)
return name, mangled_name, kind, gitmode, oid
tree_ents = (result_from_tree_entry(x) for x in tree_decode(tree_data))
if bupm:
tree_ents = sorted(tree_ents, key=lambda x: x[0])
for ent in tree_ents:
yield ent
def ordered_tree_entries(tree_data, bupm=None):
"""Yields (name, mangled_name, kind, gitmode, oid) for each item in
tree, sorted by name.
"""
# Sadly, the .bupm entries currently aren't in git tree order,
# but in unmangled name order. They _do_ account for the fact
# that git sorts trees (including chunked trees) as if their
# names ended with "/" (so "fo" sorts after "fo." iff fo is a
# directory), but we apply this on the unmangled names in save
# rather than on the mangled names.
# This makes streaming impossible when we need the metadata.
def result_from_tree_entry(tree_entry):
gitmode, mangled_name, oid = tree_entry
name, kind = git.demangle_name(mangled_name, gitmode)
return name, mangled_name, kind, gitmode, oid
tree_ents = (result_from_tree_entry(x) for x in tree_decode(tree_data))
if bupm:
tree_ents = sorted(tree_ents, key=lambda x: x[0])
for ent in tree_ents:
yield ent
def tree_data_and_bupm(repo, oid):
"""Return (tree_bytes, bupm_oid) where bupm_oid will be None if the
tree has no metadata (i.e. older bup save, or non-bup tree).
"""
assert len(oid) == 20
it = repo.cat(hexlify(oid))
_, item_t, size = next(it)
data = b''.join(it)
if item_t == b'commit':
commit = parse_commit(data)
it = repo.cat(commit.tree)
_, item_t, size = next(it)
data = b''.join(it)
assert item_t == b'tree'
elif item_t != b'tree':
raise Exception('%s is not a tree or commit' % hexstr(oid))
for _, mangled_name, sub_oid in tree_decode(data):
if mangled_name == b'.bupm':
return data, sub_oid
if mangled_name > b'.bupm':
break
return data, None
def tree_data_and_bupm(repo, oid):
"""Return (tree_bytes, bupm_oid) where bupm_oid will be None if the
tree has no metadata (i.e. older bup save, or non-bup tree).
"""
assert len(oid) == 20
it = repo.cat(oid.encode('hex'))
_, item_t, size = next(it)
data = ''.join(it)
if item_t == 'commit':
commit = parse_commit(data)
it = repo.cat(commit.tree)
_, item_t, size = next(it)
data = ''.join(it)
assert item_t == 'tree'
elif item_t != 'tree':
raise Exception('%r is not a tree or commit' % oid.encode('hex'))
for _, mangled_name, sub_oid in tree_decode(data):
if mangled_name == '.bupm':
return data, sub_oid
if mangled_name > '.bupm':
break
return data, None
def _mksubs(self):
self._subs = {}
it = cp(self._repo_dir).get(self.hash.encode("hex"))
type = it.next()
if type == "commit":
del it
it = cp(self._repo_dir).get(self.hash.encode("hex") + ":")
type = it.next()
assert type == "tree"
for (mode, mangled_name, sha) in git.tree_decode("".join(it)):
if mangled_name == ".bupm":
bupmode = stat.S_ISDIR(mode) and BUP_CHUNKED or BUP_NORMAL
self._bupm = File(self, mangled_name, GIT_MODE_FILE, sha, bupmode)
continue
(name, bupmode) = git.demangle_name(mangled_name)
if bupmode == git.BUP_CHUNKED:
mode = GIT_MODE_FILE
if stat.S_ISDIR(mode):
self._subs[name] = Dir(self, name, mode, sha, self._repo_dir)
elif stat.S_ISLNK(mode):
self._subs[name] = Symlink(self, name, sha, bupmode, self._repo_dir)
else:
self._subs[name] = File(self, name, mode, sha, bupmode, self._repo_dir)
def _treeget(hash):
it = cp().get(hash.encode('hex'))
type = it.next()
assert (type == 'tree')
return git.tree_decode(''.join(it))
def _treeget(hash, repo_dir=None):
it = cp(repo_dir).get(hash.encode("hex"))
type = it.next()
assert type == "tree"
return git.tree_decode("".join(it))
def _treeget(hash, repo_dir=None):
it = cp(repo_dir).get(hash.encode('hex'))
type = it.next()
assert (type == 'tree')
return git.tree_decode(''.join(it))
def _treeget(hash):
it = cp().get(hash.encode('hex'))
type = it.next()
assert(type == 'tree')
return git.tree_decode(''.join(it))
def _treeget(hash, repo_dir=None):
it = cp(repo_dir).get(hash.encode('hex'))
type = it.next()
assert(type == 'tree')
return git.tree_decode(''.join(it))
