def tree_items(oid, tree_data, names=frozenset(), bupm=None): def tree_item(ent_oid, kind, gitmode): if kind == BUP_CHUNKED: meta = Metadata.read(bupm) if bupm else default_file_mode return Chunky(oid=ent_oid, meta=meta) if S_ISDIR(gitmode): # No metadata here (accessable via '.' inside ent_oid). return Item(meta=default_dir_mode, oid=ent_oid) meta = Metadata.read(bupm) if bupm else None # handle the case of metadata being empty/missing in bupm # (or there not being bupm at all) if meta is None: meta = _default_mode_for_gitmode(gitmode) return Item(oid=ent_oid, meta=meta) assert len(oid) == 20 if not names: dot_meta = _read_dir_meta(bupm) if bupm else default_dir_mode yield b'.', Item(oid=oid, meta=dot_meta) tree_entries = ordered_tree_entries(tree_data, bupm) for name, mangled_name, kind, gitmode, ent_oid in tree_entries: if mangled_name == b'.bupm': continue assert name != b'.' yield name, tree_item(ent_oid, kind, gitmode) return # Assumes the tree is properly formed, i.e. there are no # duplicates, and entries will be in git tree order. if isinstance(names, (frozenset, set)): names = frozenset(names) remaining = len(names) # Account for the bupm sort order issue (cf. ordered_tree_entries above) last_name = max(names) if bupm else max(names) + b'/' if b'.' in names: dot_meta = _read_dir_meta(bupm) if bupm else default_dir_mode yield b'.', Item(oid=oid, meta=dot_meta) if remaining == 1: return remaining -= 1 tree_entries = ordered_tree_entries(tree_data, bupm) for name, mangled_name, kind, gitmode, ent_oid in tree_entries: if mangled_name == b'.bupm': continue assert name != b'.' if name not in names: if name > last_name: break # given bupm sort order, we're finished if (kind == BUP_CHUNKED or not S_ISDIR(gitmode)) and bupm: Metadata.read(bupm) continue yield name, tree_item(ent_oid, kind, gitmode) if remaining == 1: break remaining -= 1
def augment_item_meta(repo, item, include_size=False): """Ensure item has a Metadata instance for item.meta. If item.meta is currently a mode, replace it with a compatible "fake" Metadata instance. If include_size is true, ensure item.meta.size is correct, computing it if needed. If item.meta is a Metadata instance, this call may modify it in place or replace it. """ # If we actually had parallelism, we'd need locking... assert repo m = item.meta if isinstance(m, Metadata): if include_size and m.size is None: m.size = _compute_item_size(repo, item) return item._replace(meta=m) return item # m is mode meta = Metadata() meta.mode = m meta.uid = meta.gid = meta.atime = meta.mtime = meta.ctime = 0 if S_ISLNK(m): if isinstance(item, FakeLink): target = item.target else: target = _readlink(repo, item.oid) meta.symlink_target = target meta.size = len(target) elif include_size: meta.size = _compute_item_size(repo, item) return item._replace(meta=meta)
def tree_items(oid, tree_data, names=frozenset(), bupm=None): def tree_item(ent_oid, kind, gitmode): if kind == BUP_CHUNKED: meta = Metadata.read(bupm) if bupm else default_file_mode return Chunky(oid=ent_oid, meta=meta) if S_ISDIR(gitmode): # No metadata here (accessable via '.' inside ent_oid). return Item(meta=default_dir_mode, oid=ent_oid) return Item(oid=ent_oid, meta=(Metadata.read(bupm) if bupm \ else _default_mode_for_gitmode(gitmode))) assert len(oid) == 20 if not names: dot_meta = _read_dir_meta(bupm) if bupm else default_dir_mode yield '.', Item(oid=oid, meta=dot_meta) tree_entries = ordered_tree_entries(tree_data, bupm) for name, mangled_name, kind, gitmode, ent_oid in tree_entries: if mangled_name == '.bupm': continue assert name != '.' yield name, tree_item(ent_oid, kind, gitmode) return # Assumes the tree is properly formed, i.e. there are no # duplicates, and entries will be in git tree order. if type(names) not in (frozenset, set): names = frozenset(names) remaining = len(names) # Account for the bupm sort order issue (cf. ordered_tree_entries above) last_name = max(names) if bupm else max(names) + '/' if '.' in names: dot_meta = _read_dir_meta(bupm) if bupm else default_dir_mode yield '.', Item(oid=oid, meta=dot_meta) if remaining == 1: return remaining -= 1 tree_entries = ordered_tree_entries(tree_data, bupm) for name, mangled_name, kind, gitmode, ent_oid in tree_entries: if mangled_name == '.bupm': continue assert name != '.' if name not in names: if name > last_name: break # given bupm sort order, we're finished if (kind == BUP_CHUNKED or not S_ISDIR(gitmode)) and bupm: Metadata.read(bupm) continue yield name, tree_item(ent_oid, kind, gitmode) if remaining == 1: break remaining -= 1
def tree_item(ent_oid, kind, gitmode): if kind == BUP_CHUNKED: meta = Metadata.read(bupm) if bupm else default_file_mode return Chunky(oid=ent_oid, meta=meta) if S_ISDIR(gitmode): # No metadata here (accessable via '.' inside ent_oid). return Item(meta=default_dir_mode, oid=ent_oid) return Item(oid=ent_oid, meta=(Metadata.read(bupm) if bupm \ else _default_mode_for_gitmode(gitmode)))
def write_item(port, item): kind = type(item) name = bytes(kind.__name__.encode('ascii')) meta = item.meta has_meta = 1 if isinstance(meta, Metadata) else 0 if kind in (Item, Chunky, RevList): assert len(item.oid) == 20 if has_meta: vint.send(port, 'sVs', name, has_meta, item.oid) Metadata.write(meta, port, include_path=False) else: vint.send(port, 'sVsV', name, has_meta, item.oid, item.meta) elif kind in (Root, Tags): if has_meta: vint.send(port, 'sV', name, has_meta) Metadata.write(meta, port, include_path=False) else: vint.send(port, 'sVV', name, has_meta, item.meta) elif kind == Commit: assert len(item.oid) == 20 assert len(item.coid) == 20 if has_meta: vint.send(port, 'sVss', name, has_meta, item.oid, item.coid) Metadata.write(meta, port, include_path=False) else: vint.send(port, 'sVssV', name, has_meta, item.oid, item.coid, item.meta) elif kind == FakeLink: if has_meta: vint.send(port, 'sVs', name, has_meta, item.target) Metadata.write(meta, port, include_path=False) else: vint.send(port, 'sVsV', name, has_meta, item.target, item.meta) else: assert False
def write_item(port, item): kind = type(item) name = bytes(kind.__name__) meta = item.meta has_meta = 1 if isinstance(meta, Metadata) else 0 if kind in (Item, Chunky, RevList): assert len(item.oid) == 20 if has_meta: vint.send(port, 'sVs', name, has_meta, item.oid) Metadata.write(meta, port, include_path=False) else: vint.send(port, 'sVsV', name, has_meta, item.oid, item.meta) elif kind in (Root, Tags): if has_meta: vint.send(port, 'sV', name, has_meta) Metadata.write(meta, port, include_path=False) else: vint.send(port, 'sVV', name, has_meta, item.meta) elif kind == Commit: assert len(item.oid) == 20 assert len(item.coid) == 20 if has_meta: vint.send(port, 'sVss', name, has_meta, item.oid, item.coid) Metadata.write(meta, port, include_path=False) else: vint.send(port, 'sVssV', name, has_meta, item.oid, item.coid, item.meta) elif kind == FakeLink: if has_meta: vint.send(port, 'sVs', name, has_meta, item.target) Metadata.write(meta, port, include_path=False) else: vint.send(port, 'sVsV', name, has_meta, item.target, item.meta) else: assert False
def tree_item(ent_oid, kind, gitmode): if kind == BUP_CHUNKED: meta = Metadata.read(bupm) if bupm else default_file_mode return Chunky(oid=ent_oid, meta=meta) if S_ISDIR(gitmode): # No metadata here (accessable via '.' inside ent_oid). return Item(meta=default_dir_mode, oid=ent_oid) meta = Metadata.read(bupm) if bupm else None # handle the case of metadata being empty/missing in bupm # (or there not being bupm at all) if meta is None: meta = _default_mode_for_gitmode(gitmode) return Item(oid=ent_oid, meta=meta)
def tree_items(repo, oid): """Yield (name, entry_oid, meta) for each entry in oid. meta will be a Metadata object for any non-directories and for '.', otherwise None. """ # This is a simpler approach than the one in the vfs, used to # cross-check its behavior. tree_data, bupm_oid = vfs.tree_data_and_bupm(repo, oid) bupm = vfs._FileReader(repo, bupm_oid) if bupm_oid else None try: maybe_meta = lambda: Metadata.read(bupm) if bupm else None m = maybe_meta() if m: m.size = 0 yield TreeDictValue(name='.', oid=oid, meta=m) tree_ents = vfs.ordered_tree_entries(tree_data, bupm=True) for name, mangled_name, kind, gitmode, sub_oid in tree_ents: if mangled_name == '.bupm': continue assert name != '.' if S_ISDIR(gitmode): if kind == BUP_CHUNKED: yield TreeDictValue(name=name, oid=sub_oid, meta=maybe_meta()) else: yield TreeDictValue(name=name, oid=sub_oid, meta=vfs.default_dir_mode) else: yield TreeDictValue(name=name, oid=sub_oid, meta=maybe_meta()) finally: if bupm: bupm.close()
def tree_items(repo, oid): """Yield (name, entry_oid, meta) for each entry in oid. meta will be a Metadata object for any non-directories and for '.', otherwise None. """ # This is a simpler approach than the one in the vfs, used to # cross-check its behavior. tree_data, bupm_oid = vfs.tree_data_and_bupm(repo, oid) bupm = vfs._FileReader(repo, bupm_oid) if bupm_oid else None try: maybe_meta = lambda : Metadata.read(bupm) if bupm else None m = maybe_meta() if m and m.size is None: m.size = 0 yield TreeDictValue(name='.', oid=oid, meta=m) tree_ents = vfs.ordered_tree_entries(tree_data, bupm=True) for name, mangled_name, kind, gitmode, sub_oid in tree_ents: if mangled_name == '.bupm': continue assert name != '.' if S_ISDIR(gitmode): if kind == BUP_CHUNKED: yield TreeDictValue(name=name, oid=sub_oid, meta=maybe_meta()) else: yield TreeDictValue(name=name, oid=sub_oid, meta=vfs.default_dir_mode) else: yield TreeDictValue(name=name, oid=sub_oid, meta=maybe_meta()) finally: if bupm: bupm.close()
def _read_dir_meta(bupm): # This is because save writes unmodified Metadata() entries for # fake parents -- test-save-strip-graft.sh demonstrates. m = Metadata.read(bupm) if not m: return default_dir_mode assert m.mode is not None if m.size is None: m.size = 0 return m
def augment_item_meta(repo, item, include_size=False): """Ensure item has a Metadata instance for item.meta. If item.meta is currently a mode, replace it with a compatible "fake" Metadata instance. If include_size is true, ensure item.meta.size is correct, computing it if needed. If item.meta is a Metadata instance, this call may modify it in place or replace it. """ # If we actually had parallelism, we'd need locking... assert repo m = item.meta if isinstance(m, Metadata): if include_size and m.size is None: m.size = _compute_item_size(repo, item) return item._replace(meta=m) return item # m is mode meta = Metadata() meta.mode = m meta.uid = meta.gid = None meta.atime = meta.mtime = meta.ctime = 0 if S_ISLNK(m): if isinstance(item, FakeLink): target = item.target else: target = _readlink(repo, item.oid) meta.symlink_target = target meta.size = len(target) elif include_size: meta.size = _compute_item_size(repo, item) return item._replace(meta=meta)
def _write_tree(repo, dir_meta, items, omit_meta=False): if not omit_meta: if dir_meta is None: dir_meta = Metadata() metalist = [(b'', dir_meta)] metalist += [(shalist_item_sort_key((entry.mode, entry.name, None)), entry.meta) for entry in items if entry.mode != GIT_MODE_TREE] metalist.sort(key = lambda x: x[0]) metadata = BytesIO(b''.join(m[1].encode() for m in metalist)) mode, oid = split_to_blob_or_tree(repo.write_bupm, repo.write_tree, [metadata], keep_boundaries=False) shalist = [(mode, b'.bupm', oid)] else: shalist = [] shalist += [(entry.gitmode, entry.mangled_name, entry.oid) for entry in items] return repo.write_tree(shalist)
def test_resolve(): with no_lingering_errors(): with test_tempdir('bup-tvfs-') as tmpdir: resolve = vfs.resolve lresolve = vfs.lresolve bup_dir = tmpdir + '/bup' environ['GIT_DIR'] = bup_dir environ['BUP_DIR'] = bup_dir git.repodir = bup_dir data_path = tmpdir + '/src' save_time = 100000 save_time_str = strftime('%Y-%m-%d-%H%M%S', localtime(save_time)) os.mkdir(data_path) with open(data_path + '/file', 'w+') as tmpfile: print('canary', file=tmpfile) symlink('file', data_path + '/symlink') ex((bup_path, 'init')) ex((bup_path, 'index', '-v', data_path)) ex((bup_path, 'save', '-d', str(save_time), '-tvvn', 'test', '--strip', data_path)) ex((bup_path, 'tag', 'test-tag', 'test')) repo = LocalRepo() tip_hash = exo(('git', 'show-ref', 'refs/heads/test'))[0] tip_oidx = tip_hash.strip().split()[0] tip_oid = tip_oidx.decode('hex') tip_meta = Metadata() tip_meta.mode = S_IFDIR | 0o755 tip_meta.uid = tip_meta.gid = tip_meta.size = 0 tip_meta.atime = tip_meta.mtime = tip_meta.ctime = save_time * 10**9 test_revlist = vfs.RevList(meta=tip_meta, oid=tip_oid) tip_tree_oidx = exo( ('git', 'log', '--pretty=%T', '-n1', tip_oidx))[0].strip() tip_tree_oid = tip_tree_oidx.decode('hex') tip_tree = tree_dict(repo, tip_tree_oid) wvstart('resolve: /') res = resolve(repo, '/') wvpasseq(1, len(res)) wvpasseq((('', vfs._root), ), res) ignore, root_item = res[0] root_content = frozenset(vfs.contents(repo, root_item)) wvpasseq( frozenset([('.', root_item), ('.tag', vfs._tags), ('test', test_revlist)]), root_content) wvstart('resolve: /.tag') res = resolve(repo, '/.tag') wvpasseq(2, len(res)) wvpasseq((('', vfs._root), ('.tag', vfs._tags)), res) ignore, tag_item = res[1] tag_content = frozenset(vfs.contents(repo, tag_item)) wvpasseq(frozenset([('.', tag_item), ('test-tag', test_revlist)]), tag_content) wvstart('resolve: /test') res = resolve(repo, '/test') wvpasseq(2, len(res)) wvpasseq((('', vfs._root), ('test', test_revlist)), res) ignore, test_item = res[1] test_content = frozenset(vfs.contents(repo, test_item)) expected_latest_item = vfs.Item(meta=S_IFDIR | 0o755, oid=tip_tree_oid) wvpasseq( frozenset([('.', test_revlist), (save_time_str, expected_latest_item), ('latest', expected_latest_item)]), test_content) wvstart('resolve: /test/latest') res = resolve(repo, '/test/latest') wvpasseq(3, len(res)) expected_latest_item_w_meta = vfs.Item(meta=tip_tree['.'].meta, oid=tip_tree_oid) expected = (('', vfs._root), ('test', test_revlist), ('latest', expected_latest_item_w_meta)) wvpasseq(expected, res) ignore, latest_item = res[2] latest_content = frozenset(vfs.contents(repo, latest_item)) expected = frozenset( (x.name, vfs.Item(oid=x.oid, meta=x.meta)) for x in (tip_tree[name] for name in ('.', 'file', 'symlink'))) wvpasseq(expected, latest_content) wvstart('resolve: /test/latest/foo') res = resolve(repo, '/test/latest/file') wvpasseq(4, len(res)) expected_file_item_w_meta = vfs.Item(meta=tip_tree['file'].meta, oid=tip_tree['file'].oid) expected = (('', vfs._root), ('test', test_revlist), ('latest', expected_latest_item_w_meta), ('file', expected_file_item_w_meta)) wvpasseq(expected, res) wvstart('resolve: /test/latest/symlink') res = resolve(repo, '/test/latest/symlink') wvpasseq(4, len(res)) expected = (('', vfs._root), ('test', test_revlist), ('latest', expected_latest_item_w_meta), ('file', expected_file_item_w_meta)) wvpasseq(expected, res) wvstart('lresolve: /test/latest/symlink') res = lresolve(repo, '/test/latest/symlink') wvpasseq(4, len(res)) symlink_value = tip_tree['symlink'] expected_symlink_item_w_meta = vfs.Item(meta=symlink_value.meta, oid=symlink_value.oid) expected = (('', vfs._root), ('test', test_revlist), ('latest', expected_latest_item_w_meta), ('symlink', expected_symlink_item_w_meta)) wvpasseq(expected, res) wvstart('resolve: /test/latest/missing') res = resolve(repo, '/test/latest/missing') wvpasseq(4, len(res)) name, item = res[-1] wvpasseq('missing', name) wvpass(item is None)
def read_m(port, has_meta): if has_meta: m = Metadata.read(port) return m return read_vuint(port)
def meta(self): if self._meta is not None: return self._meta return Metadata()
def _commit_meta_from_auth_sec(author_sec): m = Metadata() m.mode = default_dir_mode m.uid = m.gid = m.size = 0 m.atime = m.mtime = m.ctime = author_sec * 10**9 return m