def _mksubs(self): self._subs = {} it = cp(self._repo_dir).get(self.hash.encode('hex')) type = it.next() if type == 'commit': del it it = cp(self._repo_dir).get(self.hash.encode('hex') + ':') type = it.next() assert(type == 'tree') for (mode,mangled_name,sha) in git.tree_decode(''.join(it)): if mangled_name == '.bupm': bupmode = stat.S_ISDIR(mode) and BUP_CHUNKED or BUP_NORMAL self._bupm = File(self, mangled_name, GIT_MODE_FILE, sha, bupmode) continue name, bupmode = git.demangle_name(mangled_name, mode) if bupmode == git.BUP_CHUNKED: mode = GIT_MODE_FILE if stat.S_ISDIR(mode): self._subs[name] = Dir(self, name, mode, sha, self._repo_dir) elif stat.S_ISLNK(mode): self._subs[name] = Symlink(self, name, sha, bupmode, self._repo_dir) else: self._subs[name] = File(self, name, mode, sha, bupmode, self._repo_dir)
def __init__(self, repo, oid, startofs): it = repo.cat(oid.encode('hex')) _, obj_t, size = next(it) isdir = obj_t == 'tree' data = ''.join(it) if isdir: self.it = _tree_chunks(repo, tree_decode(data), startofs) self.blob = None else: self.it = None self.blob = data[startofs:] self.ofs = startofs
def _normal_or_chunked_file_size(repo, oid): """Return the size of the normal or chunked file indicated by oid.""" # FIXME: --batch-format CatPipe? it = repo.cat(oid.encode('hex')) _, obj_t, size = next(it) ofs = 0 while obj_t == 'tree': mode, name, last_oid = last(tree_decode(''.join(it))) ofs += int(name, 16) it = repo.cat(last_oid.encode('hex')) _, obj_t, size = next(it) return ofs + sum(len(b) for b in it)
def tree_items_with_meta(repo, oid, tree_data, names): # For now, the .bupm order doesn't quite match git's, and we don't # load the tree data incrementally anyway, so we just work in RAM # via tree_data. assert len(oid) == 20 bupm = None for _, mangled_name, sub_oid in tree_decode(tree_data): if mangled_name == '.bupm': bupm = _FileReader(repo, sub_oid) break if mangled_name > '.bupm': break for item in tree_items(oid, tree_data, names, bupm): yield item
def walk_object(cat_pipe, id, verbose=None, parent_path=[], writer=None): # Yield everything reachable from id via cat_pipe, stopping # whenever we hit something writer already has. Produce (id, type # data) for each item. Since maybe_write() can't accept an # iterator, join()ing the data here doesn't hurt anything. item_it = cat_pipe.get(id) type = item_it.next() data = ''.join(item_it) id = git.calc_hash(type, data) if writer and writer.exists(id): return if type == 'blob': yield (id, type, data) elif type == 'commit': yield (id, type, data) commit_items = parse_commit(data) tree_id = commit_items.tree for x in walk_object(cat_pipe, tree_id, verbose, parent_path, writer): yield x parents = commit_items.parents for pid in parents: for x in walk_object(cat_pipe, pid, verbose, parent_path, writer): yield x elif type == 'tree': yield (id, type, data) for (mode, name, ent_id) in git.tree_decode(data): if not verbose > 1: for x in walk_object(cat_pipe, ent_id.encode('hex'), writer=writer): yield x else: demangled, bup_type = git.demangle_name(name) sub_path = parent_path + [demangled] # Don't print the sub-parts of chunked files. sub_v = verbose if bup_type == git.BUP_NORMAL else None for x in walk_object(cat_pipe, ent_id.encode('hex'), sub_v, sub_path, writer): yield x if stat.S_ISDIR(mode): if verbose > 1 and bup_type == git.BUP_NORMAL: log('%s/\n' % '/'.join(sub_path)) elif verbose > 2: # (and BUP_CHUNKED) log('%s\n' % '/'.join(sub_path)) elif verbose > 2: log('%s\n' % '/'.join(sub_path)) else: raise Exception('unexpected repository object type %r' % type)
def _tree_chunks(repo, tree, startofs): "Tree should be a sequence of (name, mode, hash) as per tree_decode()." assert(startofs >= 0) # name is the chunk's hex offset in the original file for mode, name, oid in _skip_chunks_before_offset(tree, startofs): ofs = int(name, 16) skipmore = startofs - ofs if skipmore < 0: skipmore = 0 it = repo.cat(oid.encode('hex')) _, obj_t, size = next(it) data = ''.join(it) if S_ISDIR(mode): assert obj_t == 'tree' for b in _tree_chunks(repo, tree_decode(data), skipmore): yield b else: assert obj_t == 'blob' yield data[skipmore:]
def _tree_chunks(repo, tree, startofs): "Tree should be a sequence of (name, mode, hash) as per tree_decode()." assert (startofs >= 0) # name is the chunk's hex offset in the original file for mode, name, oid in _skip_chunks_before_offset(tree, startofs): ofs = int(name, 16) skipmore = startofs - ofs if skipmore < 0: skipmore = 0 it = repo.cat(hexlify(oid)) _, obj_t, size = next(it) data = b''.join(it) if S_ISDIR(mode): assert obj_t == b'tree' for b in _tree_chunks(repo, tree_decode(data), skipmore): yield b else: assert obj_t == b'blob' yield data[skipmore:]
def _mksubs(self): self._subs = {} it = cp().get(self.hash.encode('hex')) type = it.next() if type == 'commit': del it it = cp().get(self.hash.encode('hex') + ':') type = it.next() assert(type == 'tree') for (mode,mangled_name,sha) in git.tree_decode(''.join(it)): name = mangled_name (name,bupmode) = git.demangle_name(mangled_name) if bupmode == git.BUP_CHUNKED: mode = GIT_MODE_FILE if stat.S_ISDIR(mode): self._subs[name] = Dir(self, name, mode, sha) elif stat.S_ISLNK(mode): self._subs[name] = Symlink(self, name, sha, bupmode) else: self._subs[name] = File(self, name, mode, sha, bupmode)
def _tree_chunks(repo, tree, startofs): "Tree should be a sequence of (name, mode, hash) as per tree_decode()." assert(startofs >= 0) # name is the chunk's hex offset in the original file tree = dropwhile(lambda (_1, name, _2): int(name, 16) < startofs, tree) for mode, name, oid in tree: ofs = int(name, 16) skipmore = startofs - ofs if skipmore < 0: skipmore = 0 it = repo.cat(oid.encode('hex')) _, obj_t, size = next(it) data = ''.join(it) if S_ISDIR(mode): assert obj_t == 'tree' for b in _tree_chunks(repo, tree_decode(data), skipmore): yield b else: assert obj_t == 'blob' yield data[skipmore:]
def ordered_tree_entries(tree_data, bupm=None): """Yields (name, mangled_name, kind, gitmode, oid) for each item in tree, sorted by name. """ # Sadly, the .bupm entries currently aren't in git tree order, # i.e. they don't account for the fact that git sorts trees # (including our chunked trees) as if their names ended with "/", # so "fo" sorts after "fo." iff fo is a directory. This makes # streaming impossible when we need the metadata. def result_from_tree_entry(tree_entry): gitmode, mangled_name, oid = tree_entry name, kind = git.demangle_name(mangled_name, gitmode) return name, mangled_name, kind, gitmode, oid tree_ents = (result_from_tree_entry(x) for x in tree_decode(tree_data)) if bupm: tree_ents = sorted(tree_ents, key=lambda x: x[0]) for ent in tree_ents: yield ent
def _mksubs(self): self._subs = {} it = cp().get(self.hash.encode('hex')) type = it.next() if type == 'commit': del it it = cp().get(self.hash.encode('hex') + ':') type = it.next() assert (type == 'tree') for (mode, mangled_name, sha) in git.tree_decode(''.join(it)): name = mangled_name (name, bupmode) = git.demangle_name(mangled_name) if bupmode == git.BUP_CHUNKED: mode = GIT_MODE_FILE if stat.S_ISDIR(mode): self._subs[name] = Dir(self, name, mode, sha) elif stat.S_ISLNK(mode): self._subs[name] = Symlink(self, name, sha, bupmode) else: self._subs[name] = File(self, name, mode, sha, bupmode)
def ordered_tree_entries(tree_data, bupm=None): """Yields (name, mangled_name, kind, gitmode, oid) for each item in tree, sorted by name. """ # Sadly, the .bupm entries currently aren't in git tree order, # but in unmangled name order. They _do_ account for the fact # that git sorts trees (including chunked trees) as if their # names ended with "/" (so "fo" sorts after "fo." iff fo is a # directory), but we apply this on the unmangled names in save # rather than on the mangled names. # This makes streaming impossible when we need the metadata. def result_from_tree_entry(tree_entry): gitmode, mangled_name, oid = tree_entry name, kind = git.demangle_name(mangled_name, gitmode) return name, mangled_name, kind, gitmode, oid tree_ents = (result_from_tree_entry(x) for x in tree_decode(tree_data)) if bupm: tree_ents = sorted(tree_ents, key=lambda x: x[0]) for ent in tree_ents: yield ent
def tree_data_and_bupm(repo, oid): """Return (tree_bytes, bupm_oid) where bupm_oid will be None if the tree has no metadata (i.e. older bup save, or non-bup tree). """ assert len(oid) == 20 it = repo.cat(hexlify(oid)) _, item_t, size = next(it) data = b''.join(it) if item_t == b'commit': commit = parse_commit(data) it = repo.cat(commit.tree) _, item_t, size = next(it) data = b''.join(it) assert item_t == b'tree' elif item_t != b'tree': raise Exception('%s is not a tree or commit' % hexstr(oid)) for _, mangled_name, sub_oid in tree_decode(data): if mangled_name == b'.bupm': return data, sub_oid if mangled_name > b'.bupm': break return data, None
def tree_data_and_bupm(repo, oid): """Return (tree_bytes, bupm_oid) where bupm_oid will be None if the tree has no metadata (i.e. older bup save, or non-bup tree). """ assert len(oid) == 20 it = repo.cat(oid.encode('hex')) _, item_t, size = next(it) data = ''.join(it) if item_t == 'commit': commit = parse_commit(data) it = repo.cat(commit.tree) _, item_t, size = next(it) data = ''.join(it) assert item_t == 'tree' elif item_t != 'tree': raise Exception('%r is not a tree or commit' % oid.encode('hex')) for _, mangled_name, sub_oid in tree_decode(data): if mangled_name == '.bupm': return data, sub_oid if mangled_name > '.bupm': break return data, None
def _mksubs(self): self._subs = {} it = cp(self._repo_dir).get(self.hash.encode("hex")) type = it.next() if type == "commit": del it it = cp(self._repo_dir).get(self.hash.encode("hex") + ":") type = it.next() assert type == "tree" for (mode, mangled_name, sha) in git.tree_decode("".join(it)): if mangled_name == ".bupm": bupmode = stat.S_ISDIR(mode) and BUP_CHUNKED or BUP_NORMAL self._bupm = File(self, mangled_name, GIT_MODE_FILE, sha, bupmode) continue (name, bupmode) = git.demangle_name(mangled_name) if bupmode == git.BUP_CHUNKED: mode = GIT_MODE_FILE if stat.S_ISDIR(mode): self._subs[name] = Dir(self, name, mode, sha, self._repo_dir) elif stat.S_ISLNK(mode): self._subs[name] = Symlink(self, name, sha, bupmode, self._repo_dir) else: self._subs[name] = File(self, name, mode, sha, bupmode, self._repo_dir)
def _treeget(hash): it = cp().get(hash.encode('hex')) type = it.next() assert (type == 'tree') return git.tree_decode(''.join(it))
def _treeget(hash, repo_dir=None): it = cp(repo_dir).get(hash.encode("hex")) type = it.next() assert type == "tree" return git.tree_decode("".join(it))
def _treeget(hash, repo_dir=None): it = cp(repo_dir).get(hash.encode('hex')) type = it.next() assert (type == 'tree') return git.tree_decode(''.join(it))
def _treeget(hash): it = cp().get(hash.encode('hex')) type = it.next() assert(type == 'tree') return git.tree_decode(''.join(it))
def _treeget(hash, repo_dir=None): it = cp(repo_dir).get(hash.encode('hex')) type = it.next() assert(type == 'tree') return git.tree_decode(''.join(it))