def tree_items(oid, tree_data, names=frozenset(), bupm=None):
def tree_item(ent_oid, kind, gitmode):
if kind == BUP_CHUNKED:
meta = Metadata.read(bupm) if bupm else default_file_mode
return Chunky(oid=ent_oid, meta=meta)
if S_ISDIR(gitmode):
# No metadata here (accessable via '.' inside ent_oid).
return Item(meta=default_dir_mode, oid=ent_oid)
meta = Metadata.read(bupm) if bupm else None
# handle the case of metadata being empty/missing in bupm
# (or there not being bupm at all)
if meta is None:
meta = _default_mode_for_gitmode(gitmode)
return Item(oid=ent_oid, meta=meta)
assert len(oid) == 20
if not names:
dot_meta = _read_dir_meta(bupm) if bupm else default_dir_mode
yield b'.', Item(oid=oid, meta=dot_meta)
tree_entries = ordered_tree_entries(tree_data, bupm)
for name, mangled_name, kind, gitmode, ent_oid in tree_entries:
if mangled_name == b'.bupm':
continue
assert name != b'.'
yield name, tree_item(ent_oid, kind, gitmode)
return
# Assumes the tree is properly formed, i.e. there are no
# duplicates, and entries will be in git tree order.
if isinstance(names, (frozenset, set)):
names = frozenset(names)
remaining = len(names)
# Account for the bupm sort order issue (cf. ordered_tree_entries above)
last_name = max(names) if bupm else max(names) + b'/'
if b'.' in names:
dot_meta = _read_dir_meta(bupm) if bupm else default_dir_mode
yield b'.', Item(oid=oid, meta=dot_meta)
if remaining == 1:
return
remaining -= 1
tree_entries = ordered_tree_entries(tree_data, bupm)
for name, mangled_name, kind, gitmode, ent_oid in tree_entries:
if mangled_name == b'.bupm':
continue
assert name != b'.'
if name not in names:
if name > last_name:
break # given bupm sort order, we're finished
if (kind == BUP_CHUNKED or not S_ISDIR(gitmode)) and bupm:
Metadata.read(bupm)
continue
yield name, tree_item(ent_oid, kind, gitmode)
if remaining == 1:
break
remaining -= 1
def augment_item_meta(repo, item, include_size=False):
"""Ensure item has a Metadata instance for item.meta. If item.meta is
currently a mode, replace it with a compatible "fake" Metadata
instance. If include_size is true, ensure item.meta.size is
correct, computing it if needed. If item.meta is a Metadata
instance, this call may modify it in place or replace it.
"""
# If we actually had parallelism, we'd need locking...
assert repo
m = item.meta
if isinstance(m, Metadata):
if include_size and m.size is None:
m.size = _compute_item_size(repo, item)
return item._replace(meta=m)
return item
# m is mode
meta = Metadata()
meta.mode = m
meta.uid = meta.gid = meta.atime = meta.mtime = meta.ctime = 0
if S_ISLNK(m):
if isinstance(item, FakeLink):
target = item.target
else:
target = _readlink(repo, item.oid)
meta.symlink_target = target
meta.size = len(target)
elif include_size:
meta.size = _compute_item_size(repo, item)
return item._replace(meta=meta)
def tree_items(oid, tree_data, names=frozenset(), bupm=None):
def tree_item(ent_oid, kind, gitmode):
if kind == BUP_CHUNKED:
meta = Metadata.read(bupm) if bupm else default_file_mode
return Chunky(oid=ent_oid, meta=meta)
if S_ISDIR(gitmode):
# No metadata here (accessable via '.' inside ent_oid).
return Item(meta=default_dir_mode, oid=ent_oid)
return Item(oid=ent_oid,
meta=(Metadata.read(bupm) if bupm \
else _default_mode_for_gitmode(gitmode)))
assert len(oid) == 20
if not names:
dot_meta = _read_dir_meta(bupm) if bupm else default_dir_mode
yield '.', Item(oid=oid, meta=dot_meta)
tree_entries = ordered_tree_entries(tree_data, bupm)
for name, mangled_name, kind, gitmode, ent_oid in tree_entries:
if mangled_name == '.bupm':
continue
assert name != '.'
yield name, tree_item(ent_oid, kind, gitmode)
return
# Assumes the tree is properly formed, i.e. there are no
# duplicates, and entries will be in git tree order.
if type(names) not in (frozenset, set):
names = frozenset(names)
remaining = len(names)
# Account for the bupm sort order issue (cf. ordered_tree_entries above)
last_name = max(names) if bupm else max(names) + '/'
if '.' in names:
dot_meta = _read_dir_meta(bupm) if bupm else default_dir_mode
yield '.', Item(oid=oid, meta=dot_meta)
if remaining == 1:
return
remaining -= 1
tree_entries = ordered_tree_entries(tree_data, bupm)
for name, mangled_name, kind, gitmode, ent_oid in tree_entries:
if mangled_name == '.bupm':
continue
assert name != '.'
if name not in names:
if name > last_name:
break # given bupm sort order, we're finished
if (kind == BUP_CHUNKED or not S_ISDIR(gitmode)) and bupm:
Metadata.read(bupm)
continue
yield name, tree_item(ent_oid, kind, gitmode)
if remaining == 1:
break
remaining -= 1
def tree_item(ent_oid, kind, gitmode):
if kind == BUP_CHUNKED:
meta = Metadata.read(bupm) if bupm else default_file_mode
return Chunky(oid=ent_oid, meta=meta)
if S_ISDIR(gitmode):
# No metadata here (accessable via '.' inside ent_oid).
return Item(meta=default_dir_mode, oid=ent_oid)
return Item(oid=ent_oid,
meta=(Metadata.read(bupm) if bupm \
else _default_mode_for_gitmode(gitmode)))
def tree_item(ent_oid, kind, gitmode):
if kind == BUP_CHUNKED:
meta = Metadata.read(bupm) if bupm else default_file_mode
return Chunky(oid=ent_oid, meta=meta)
if S_ISDIR(gitmode):
# No metadata here (accessable via '.' inside ent_oid).
return Item(meta=default_dir_mode, oid=ent_oid)
return Item(oid=ent_oid,
meta=(Metadata.read(bupm) if bupm \
else _default_mode_for_gitmode(gitmode)))
def write_item(port, item):
kind = type(item)
name = bytes(kind.__name__.encode('ascii'))
meta = item.meta
has_meta = 1 if isinstance(meta, Metadata) else 0
if kind in (Item, Chunky, RevList):
assert len(item.oid) == 20
if has_meta:
vint.send(port, 'sVs', name, has_meta, item.oid)
Metadata.write(meta, port, include_path=False)
else:
vint.send(port, 'sVsV', name, has_meta, item.oid, item.meta)
elif kind in (Root, Tags):
if has_meta:
vint.send(port, 'sV', name, has_meta)
Metadata.write(meta, port, include_path=False)
else:
vint.send(port, 'sVV', name, has_meta, item.meta)
elif kind == Commit:
assert len(item.oid) == 20
assert len(item.coid) == 20
if has_meta:
vint.send(port, 'sVss', name, has_meta, item.oid, item.coid)
Metadata.write(meta, port, include_path=False)
else:
vint.send(port, 'sVssV', name, has_meta, item.oid, item.coid,
item.meta)
elif kind == FakeLink:
if has_meta:
vint.send(port, 'sVs', name, has_meta, item.target)
Metadata.write(meta, port, include_path=False)
else:
vint.send(port, 'sVsV', name, has_meta, item.target, item.meta)
else:
assert False
def write_item(port, item):
kind = type(item)
name = bytes(kind.__name__)
meta = item.meta
has_meta = 1 if isinstance(meta, Metadata) else 0
if kind in (Item, Chunky, RevList):
assert len(item.oid) == 20
if has_meta:
vint.send(port, 'sVs', name, has_meta, item.oid)
Metadata.write(meta, port, include_path=False)
else:
vint.send(port, 'sVsV', name, has_meta, item.oid, item.meta)
elif kind in (Root, Tags):
if has_meta:
vint.send(port, 'sV', name, has_meta)
Metadata.write(meta, port, include_path=False)
else:
vint.send(port, 'sVV', name, has_meta, item.meta)
elif kind == Commit:
assert len(item.oid) == 20
assert len(item.coid) == 20
if has_meta:
vint.send(port, 'sVss', name, has_meta, item.oid, item.coid)
Metadata.write(meta, port, include_path=False)
else:
vint.send(port, 'sVssV', name, has_meta, item.oid, item.coid,
item.meta)
elif kind == FakeLink:
if has_meta:
vint.send(port, 'sVs', name, has_meta, item.target)
Metadata.write(meta, port, include_path=False)
else:
vint.send(port, 'sVsV', name, has_meta, item.target, item.meta)
else:
assert False
def tree_item(ent_oid, kind, gitmode):
if kind == BUP_CHUNKED:
meta = Metadata.read(bupm) if bupm else default_file_mode
return Chunky(oid=ent_oid, meta=meta)
if S_ISDIR(gitmode):
# No metadata here (accessable via '.' inside ent_oid).
return Item(meta=default_dir_mode, oid=ent_oid)
meta = Metadata.read(bupm) if bupm else None
# handle the case of metadata being empty/missing in bupm
# (or there not being bupm at all)
if meta is None:
meta = _default_mode_for_gitmode(gitmode)
return Item(oid=ent_oid, meta=meta)
def tree_items(repo, oid):
"""Yield (name, entry_oid, meta) for each entry in oid. meta will be
a Metadata object for any non-directories and for '.', otherwise
None.
"""
# This is a simpler approach than the one in the vfs, used to
# cross-check its behavior.
tree_data, bupm_oid = vfs.tree_data_and_bupm(repo, oid)
bupm = vfs._FileReader(repo, bupm_oid) if bupm_oid else None
try:
maybe_meta = lambda: Metadata.read(bupm) if bupm else None
m = maybe_meta()
if m:
m.size = 0
yield TreeDictValue(name='.', oid=oid, meta=m)
tree_ents = vfs.ordered_tree_entries(tree_data, bupm=True)
for name, mangled_name, kind, gitmode, sub_oid in tree_ents:
if mangled_name == '.bupm':
continue
assert name != '.'
if S_ISDIR(gitmode):
if kind == BUP_CHUNKED:
yield TreeDictValue(name=name,
oid=sub_oid,
meta=maybe_meta())
else:
yield TreeDictValue(name=name,
oid=sub_oid,
meta=vfs.default_dir_mode)
else:
yield TreeDictValue(name=name, oid=sub_oid, meta=maybe_meta())
finally:
if bupm:
bupm.close()
def tree_items(repo, oid):
"""Yield (name, entry_oid, meta) for each entry in oid. meta will be
a Metadata object for any non-directories and for '.', otherwise
None.
"""
# This is a simpler approach than the one in the vfs, used to
# cross-check its behavior.
tree_data, bupm_oid = vfs.tree_data_and_bupm(repo, oid)
bupm = vfs._FileReader(repo, bupm_oid) if bupm_oid else None
try:
maybe_meta = lambda : Metadata.read(bupm) if bupm else None
m = maybe_meta()
if m and m.size is None:
m.size = 0
yield TreeDictValue(name='.', oid=oid, meta=m)
tree_ents = vfs.ordered_tree_entries(tree_data, bupm=True)
for name, mangled_name, kind, gitmode, sub_oid in tree_ents:
if mangled_name == '.bupm':
continue
assert name != '.'
if S_ISDIR(gitmode):
if kind == BUP_CHUNKED:
yield TreeDictValue(name=name, oid=sub_oid,
meta=maybe_meta())
else:
yield TreeDictValue(name=name, oid=sub_oid,
meta=vfs.default_dir_mode)
else:
yield TreeDictValue(name=name, oid=sub_oid, meta=maybe_meta())
finally:
if bupm:
bupm.close()
def _read_dir_meta(bupm):
# This is because save writes unmodified Metadata() entries for
# fake parents -- test-save-strip-graft.sh demonstrates.
m = Metadata.read(bupm)
if not m:
return default_dir_mode
assert m.mode is not None
if m.size is None:
m.size = 0
return m
def _read_dir_meta(bupm):
# This is because save writes unmodified Metadata() entries for
# fake parents -- test-save-strip-graft.sh demonstrates.
m = Metadata.read(bupm)
if not m:
return default_dir_mode
assert m.mode is not None
if m.size is None:
m.size = 0
return m
def augment_item_meta(repo, item, include_size=False):
"""Ensure item has a Metadata instance for item.meta. If item.meta is
currently a mode, replace it with a compatible "fake" Metadata
instance. If include_size is true, ensure item.meta.size is
correct, computing it if needed. If item.meta is a Metadata
instance, this call may modify it in place or replace it.
"""
# If we actually had parallelism, we'd need locking...
assert repo
m = item.meta
if isinstance(m, Metadata):
if include_size and m.size is None:
m.size = _compute_item_size(repo, item)
return item._replace(meta=m)
return item
# m is mode
meta = Metadata()
meta.mode = m
meta.uid = meta.gid = None
meta.atime = meta.mtime = meta.ctime = 0
if S_ISLNK(m):
if isinstance(item, FakeLink):
target = item.target
else:
target = _readlink(repo, item.oid)
meta.symlink_target = target
meta.size = len(target)
elif include_size:
meta.size = _compute_item_size(repo, item)
return item._replace(meta=meta)
def _write_tree(repo, dir_meta, items, omit_meta=False):
if not omit_meta:
if dir_meta is None:
dir_meta = Metadata()
metalist = [(b'', dir_meta)]
metalist += [(shalist_item_sort_key((entry.mode, entry.name, None)),
entry.meta)
for entry in items if entry.mode != GIT_MODE_TREE]
metalist.sort(key = lambda x: x[0])
metadata = BytesIO(b''.join(m[1].encode() for m in metalist))
mode, oid = split_to_blob_or_tree(repo.write_bupm, repo.write_tree,
[metadata],
keep_boundaries=False)
shalist = [(mode, b'.bupm', oid)]
else:
shalist = []
shalist += [(entry.gitmode, entry.mangled_name, entry.oid)
for entry in items]
return repo.write_tree(shalist)
def test_resolve():
with no_lingering_errors():
with test_tempdir('bup-tvfs-') as tmpdir:
resolve = vfs.resolve
lresolve = vfs.lresolve
bup_dir = tmpdir + '/bup'
environ['GIT_DIR'] = bup_dir
environ['BUP_DIR'] = bup_dir
git.repodir = bup_dir
data_path = tmpdir + '/src'
save_time = 100000
save_time_str = strftime('%Y-%m-%d-%H%M%S', localtime(save_time))
os.mkdir(data_path)
with open(data_path + '/file', 'w+') as tmpfile:
print('canary', file=tmpfile)
symlink('file', data_path + '/symlink')
ex((bup_path, 'init'))
ex((bup_path, 'index', '-v', data_path))
ex((bup_path, 'save', '-d', str(save_time), '-tvvn', 'test',
'--strip', data_path))
ex((bup_path, 'tag', 'test-tag', 'test'))
repo = LocalRepo()
tip_hash = exo(('git', 'show-ref', 'refs/heads/test'))[0]
tip_oidx = tip_hash.strip().split()[0]
tip_oid = tip_oidx.decode('hex')
tip_meta = Metadata()
tip_meta.mode = S_IFDIR | 0o755
tip_meta.uid = tip_meta.gid = tip_meta.size = 0
tip_meta.atime = tip_meta.mtime = tip_meta.ctime = save_time * 10**9
test_revlist = vfs.RevList(meta=tip_meta, oid=tip_oid)
tip_tree_oidx = exo(
('git', 'log', '--pretty=%T', '-n1', tip_oidx))[0].strip()
tip_tree_oid = tip_tree_oidx.decode('hex')
tip_tree = tree_dict(repo, tip_tree_oid)
wvstart('resolve: /')
res = resolve(repo, '/')
wvpasseq(1, len(res))
wvpasseq((('', vfs._root), ), res)
ignore, root_item = res[0]
root_content = frozenset(vfs.contents(repo, root_item))
wvpasseq(
frozenset([('.', root_item), ('.tag', vfs._tags),
('test', test_revlist)]), root_content)
wvstart('resolve: /.tag')
res = resolve(repo, '/.tag')
wvpasseq(2, len(res))
wvpasseq((('', vfs._root), ('.tag', vfs._tags)), res)
ignore, tag_item = res[1]
tag_content = frozenset(vfs.contents(repo, tag_item))
wvpasseq(frozenset([('.', tag_item), ('test-tag', test_revlist)]),
tag_content)
wvstart('resolve: /test')
res = resolve(repo, '/test')
wvpasseq(2, len(res))
wvpasseq((('', vfs._root), ('test', test_revlist)), res)
ignore, test_item = res[1]
test_content = frozenset(vfs.contents(repo, test_item))
expected_latest_item = vfs.Item(meta=S_IFDIR | 0o755,
oid=tip_tree_oid)
wvpasseq(
frozenset([('.', test_revlist),
(save_time_str, expected_latest_item),
('latest', expected_latest_item)]), test_content)
wvstart('resolve: /test/latest')
res = resolve(repo, '/test/latest')
wvpasseq(3, len(res))
expected_latest_item_w_meta = vfs.Item(meta=tip_tree['.'].meta,
oid=tip_tree_oid)
expected = (('', vfs._root), ('test', test_revlist),
('latest', expected_latest_item_w_meta))
wvpasseq(expected, res)
ignore, latest_item = res[2]
latest_content = frozenset(vfs.contents(repo, latest_item))
expected = frozenset(
(x.name, vfs.Item(oid=x.oid, meta=x.meta))
for x in (tip_tree[name] for name in ('.', 'file', 'symlink')))
wvpasseq(expected, latest_content)
wvstart('resolve: /test/latest/foo')
res = resolve(repo, '/test/latest/file')
wvpasseq(4, len(res))
expected_file_item_w_meta = vfs.Item(meta=tip_tree['file'].meta,
oid=tip_tree['file'].oid)
expected = (('', vfs._root), ('test', test_revlist),
('latest', expected_latest_item_w_meta),
('file', expected_file_item_w_meta))
wvpasseq(expected, res)
wvstart('resolve: /test/latest/symlink')
res = resolve(repo, '/test/latest/symlink')
wvpasseq(4, len(res))
expected = (('', vfs._root), ('test', test_revlist),
('latest', expected_latest_item_w_meta),
('file', expected_file_item_w_meta))
wvpasseq(expected, res)
wvstart('lresolve: /test/latest/symlink')
res = lresolve(repo, '/test/latest/symlink')
wvpasseq(4, len(res))
symlink_value = tip_tree['symlink']
expected_symlink_item_w_meta = vfs.Item(meta=symlink_value.meta,
oid=symlink_value.oid)
expected = (('', vfs._root), ('test', test_revlist),
('latest', expected_latest_item_w_meta),
('symlink', expected_symlink_item_w_meta))
wvpasseq(expected, res)
wvstart('resolve: /test/latest/missing')
res = resolve(repo, '/test/latest/missing')
wvpasseq(4, len(res))
name, item = res[-1]
wvpasseq('missing', name)
wvpass(item is None)
def read_m(port, has_meta):
if has_meta:
m = Metadata.read(port)
return m
return read_vuint(port)
def read_m(port, has_meta):
if has_meta:
m = Metadata.read(port)
return m
return read_vuint(port)
def meta(self):
if self._meta is not None:
return self._meta
return Metadata()
def _commit_meta_from_auth_sec(author_sec):
m = Metadata()
m.mode = default_dir_mode
m.uid = m.gid = m.size = 0
m.atime = m.mtime = m.ctime = author_sec * 10**9
return m
