def test_cache_behavior():
orig_max = vfs._cache_max_items
try:
vfs._cache_max_items = 2
vfs.clear_cache()
wvpasseq({}, vfs._cache)
wvpasseq([], vfs._cache_keys)
wvfail(vfs._cache_keys)
wvexcept(Exception, vfs.cache_notice, b'x', 1)
key_0 = b'itm:' + b'\0' * 20
key_1 = b'itm:' + b'\1' * 20
key_2 = b'itm:' + b'\2' * 20
vfs.cache_notice(key_0, b'something')
wvpasseq({key_0: b'something'}, vfs._cache)
wvpasseq([key_0], vfs._cache_keys)
vfs.cache_notice(key_1, b'something else')
wvpasseq({key_0: b'something', key_1: b'something else'}, vfs._cache)
wvpasseq(frozenset([key_0, key_1]), frozenset(vfs._cache_keys))
vfs.cache_notice(key_2, b'and also')
wvpasseq(2, len(vfs._cache))
wvpass(
frozenset(items(vfs._cache)) < frozenset(
items({
key_0: b'something',
key_1: b'something else',
key_2: b'and also'
})))
wvpasseq(2, len(vfs._cache_keys))
wvpass(frozenset(vfs._cache_keys) < frozenset([key_0, key_1, key_2]))
vfs.clear_cache()
wvpasseq({}, vfs._cache)
wvpasseq([], vfs._cache_keys)
finally:
vfs._cache_max_items = orig_max
vfs.clear_cache()
def bup_rm(repo, paths, compression=6, verbosity=None):
dead_branches, dead_saves = dead_items(repo, paths)
die_if_errors('not proceeding with any removals\n')
updated_refs = {} # ref_name -> (original_ref, tip_commit(bin))
for branchname, branchitem in compat.items(dead_branches):
ref = 'refs/heads/' + branchname
assert (not ref in updated_refs)
updated_refs[ref] = (branchitem.oid, None)
if dead_saves:
writer = git.PackWriter(compression_level=compression)
try:
for branch, saves in compat.items(dead_saves):
assert (saves)
updated_refs['refs/heads/' + branch] = rm_saves(saves, writer)
except:
if writer:
writer.abort()
raise
else:
if writer:
# Must close before we can update the ref(s) below.
writer.close()
# Only update the refs here, at the very end, so that if something
# goes wrong above, the old refs will be undisturbed. Make an attempt
# to update each ref.
for ref_name, info in compat.items(updated_refs):
orig_ref, new_ref = info
try:
if not new_ref:
git.delete_ref(ref_name, orig_ref.encode('hex'))
else:
git.update_ref(ref_name, new_ref, orig_ref)
if verbosity:
new_hex = new_ref.encode('hex')
if orig_ref:
orig_hex = orig_ref.encode('hex')
log('updated %r (%s -> %s)\n' %
(ref_name, orig_hex, new_hex))
else:
log('updated %r (%s)\n' % (ref_name, new_hex))
except (git.GitError, ClientError) as ex:
if new_ref:
add_error('while trying to update %r (%s -> %s): %s' %
(ref_name, orig_ref, new_ref, ex))
else:
add_error('while trying to delete %r (%s): %s' %
(ref_name, orig_ref, ex))
def bup_rm(repo, paths, compression=6, verbosity=None):
dead_branches, dead_saves = dead_items(repo, paths)
die_if_errors('not proceeding with any removals\n')
updated_refs = {} # ref_name -> (original_ref, tip_commit(bin))
for branchname, branchitem in compat.items(dead_branches):
ref = 'refs/heads/' + branchname
assert(not ref in updated_refs)
updated_refs[ref] = (branchitem.oid, None)
if dead_saves:
writer = git.PackWriter(compression_level=compression)
try:
for branch, saves in compat.items(dead_saves):
assert(saves)
updated_refs['refs/heads/' + branch] = rm_saves(saves, writer)
except:
if writer:
writer.abort()
raise
else:
if writer:
# Must close before we can update the ref(s) below.
writer.close()
# Only update the refs here, at the very end, so that if something
# goes wrong above, the old refs will be undisturbed. Make an attempt
# to update each ref.
for ref_name, info in compat.items(updated_refs):
orig_ref, new_ref = info
try:
if not new_ref:
git.delete_ref(ref_name, orig_ref.encode('hex'))
else:
git.update_ref(ref_name, new_ref, orig_ref)
if verbosity:
new_hex = new_ref.encode('hex')
if orig_ref:
orig_hex = orig_ref.encode('hex')
log('updated %r (%s -> %s)\n'
% (ref_name, orig_hex, new_hex))
else:
log('updated %r (%s)\n' % (ref_name, new_hex))
except (git.GitError, ClientError) as ex:
if new_ref:
add_error('while trying to update %r (%s -> %s): %s'
% (ref_name, orig_ref, new_ref, ex))
else:
add_error('while trying to delete %r (%s): %s'
% (ref_name, orig_ref, ex))
def _argmax_base(command):
base_size = 2048
for c in command:
base_size += len(command) + 1
for k, v in compat.items(environ):
base_size += len(k) + len(v) + 2 + sizeof(c_void_p)
return base_size
def expected_retentions(utcs, utc_start, spec):
if not spec:
return utcs
utcs = sorted(utcs, reverse=True)
period_start = dict(spec)
for kind, duration in compat.items(period_start):
period_start[kind] = utc_start - period_as_secs(duration)
period_start = defaultdict(lambda: float('inf'), period_start)
all = list(takewhile(lambda x: x >= period_start[b'all'], utcs))
utcs = list(dropwhile(lambda x: x >= period_start[b'all'], utcs))
matches = takewhile(lambda x: x >= period_start[b'dailies'], utcs)
dailies = [max(day_utcs) for yday, day_utcs
in groupby(matches, lambda x: localtime(x).tm_yday)]
utcs = list(dropwhile(lambda x: x >= period_start[b'dailies'], utcs))
matches = takewhile(lambda x: x >= period_start[b'monthlies'], utcs)
monthlies = [max(month_utcs) for month, month_utcs
in groupby(matches, lambda x: localtime(x).tm_mon)]
utcs = dropwhile(lambda x: x >= period_start[b'monthlies'], utcs)
matches = takewhile(lambda x: x >= period_start[b'yearlies'], utcs)
yearlies = [max(year_utcs) for year, year_utcs
in groupby(matches, lambda x: localtime(x).tm_year)]
return chain(all, dailies, monthlies, yearlies)
def __init__(self, filename):
# Map a "dev:ino" node to a list of paths associated with that node.
self._node_paths = {}
# Map a path to a "dev:ino" node.
self._path_node = {}
self._filename = filename
self._save_prepared = None
self._tmpname = None
f = None
try:
f = open(filename, 'rb')
except IOError as e:
if e.errno == errno.ENOENT:
pass
else:
raise
if f:
try:
self._node_paths = pickle_load(f)
finally:
f.close()
f = None
# Set up the reverse hard link index.
for node, paths in compat.items(self._node_paths):
for path in paths:
self._path_node[path] = node
def _argmax_base(command):
base_size = 2048
for c in command:
base_size += len(command) + 1
for k, v in compat.items(environ):
base_size += len(k) + len(v) + 2 + sizeof(c_void_p)
return base_size
def __init__(self, filename):
# Map a "dev:ino" node to a list of paths associated with that node.
self._node_paths = {}
# Map a path to a "dev:ino" node.
self._path_node = {}
self._filename = filename
self._save_prepared = None
self._tmpname = None
f = None
try:
f = open(filename, 'r')
except IOError as e:
if e.errno == errno.ENOENT:
pass
else:
raise
if f:
try:
self._node_paths = cPickle.load(f)
finally:
f.close()
f = None
# Set up the reverse hard link index.
for node, paths in compat.items(self._node_paths):
for path in paths:
self._path_node[path] = node
def verify_only_refs(**kwargs):
for kind, refs in items(kwargs):
if kind == 'heads':
abs_refs = [b'refs/heads/' + ref for ref in refs]
karg = b'--heads'
elif kind == 'tags':
abs_refs = [b'refs/tags/' + ref for ref in refs]
karg = b'--tags'
else:
raise TypeError('unexpected keyword argument %r' % kind)
if abs_refs:
verify_rcz([
b'git', b'--git-dir', b'get-dest', b'show-ref', b'--verify',
karg
] + abs_refs)
exr = exo((b'git', b'--git-dir', b'get-dest', b'show-ref', karg),
check=False)
wvpasseq(0, exr.rc)
expected_refs = sorted(abs_refs)
repo_refs = sorted([x.split()[1] for x in exr.out.splitlines()])
wvpasseq(expected_refs, repo_refs)
else:
# FIXME: can we just check "git show-ref --heads == ''"?
exr = exo((b'git', b'--git-dir', b'get-dest', b'show-ref', karg),
check=False)
wvpasseq(1, exr.rc)
wvpasseq(b'', exr.out.strip())
def bup_rm(repo, paths, verbosity=None):
dead_branches, dead_saves = dead_items(repo, paths)
die_if_errors('not proceeding with any removals\n')
updated_refs = {} # ref_name -> (original_ref, tip_commit(bin))
for branchname, branchitem in compat.items(dead_branches):
ref = b'refs/heads/' + branchname
assert (not ref in updated_refs)
updated_refs[ref] = (branchitem.oid, None)
if dead_saves:
try:
for branch, saves in compat.items(dead_saves):
assert (saves)
updated_refs[b'refs/heads/' + branch] = rm_saves(repo, saves)
except BaseException as ex:
with pending_raise(ex):
repo.abort_writing()
# Only update the refs here, at the very end, so that if something
# goes wrong above, the old refs will be undisturbed. Make an attempt
# to update each ref.
for ref_name, info in compat.items(updated_refs):
orig_ref, new_ref = info
try:
if not new_ref:
repo.delete_ref(ref_name, orig_ref)
else:
repo.update_ref(ref_name, new_ref, orig_ref)
if verbosity:
log('updated %s (%s%s)\n' %
(path_msg(ref_name), hexstr(orig_ref) +
' -> ' if orig_ref else '', hexstr(new_ref)))
except (git.GitError, ClientError) as ex:
if new_ref:
add_error('while trying to update %s (%s%s): %s' %
(path_msg(ref_name), hexstr(orig_ref) +
' -> ' if orig_ref else '', hexstr(new_ref), ex))
else:
add_error('while trying to delete %r (%s): %s' %
(ref_name, hexstr(orig_ref), ex))
def filter_output(srcs, dests):
"""Transfer data from file descriptors in srcs to the corresponding
file descriptors in dests print_clean_line until all of the srcs
have closed.
"""
global sep_rx
assert all(type(x) in int_types for x in srcs)
assert all(type(x) in int_types for x in srcs)
assert len(srcs) == len(dests)
srcs = tuple(srcs)
dest_for = dict(zip(srcs, dests))
pending = {}
pending_ex = None
try:
while srcs:
ready_fds, _, _ = select.select(srcs, [], [])
width = tty_width()
for fd in ready_fds:
buf = os.read(fd, 4096)
dest = dest_for[fd]
if not buf:
srcs = tuple([x for x in srcs if x is not fd])
print_clean_line(dest, pending.pop(fd, []), width)
else:
split = sep_rx.split(buf)
while len(split) > 1:
content, sep = split[:2]
split = split[2:]
print_clean_line(dest,
pending.pop(fd, []) + [content],
width,
sep)
assert len(split) == 1
if split[0]:
pending.setdefault(fd, []).extend(split)
except BaseException as ex:
pending_ex = add_ex_ctx(add_ex_tb(ex), pending_ex)
try:
# Try to finish each of the streams
for fd, pending_items in compat.items(pending):
dest = dest_for[fd]
width = tty_width()
try:
print_clean_line(dest, pending_items, width)
except (EnvironmentError, EOFError) as ex:
pending_ex = add_ex_ctx(add_ex_tb(ex), pending_ex)
except BaseException as ex:
pending_ex = add_ex_ctx(add_ex_tb(ex), pending_ex)
if pending_ex:
raise pending_ex
def filter_output(src_out, src_err, dest_out, dest_err):
"""Transfer data from src_out to dest_out and src_err to dest_err via
print_clean_line until src_out and src_err close."""
global sep_rx
assert not isinstance(src_out, bool)
assert not isinstance(src_err, bool)
assert not isinstance(dest_out, bool)
assert not isinstance(dest_err, bool)
assert src_out is not None or src_err is not None
assert (src_out is None) == (dest_out is None)
assert (src_err is None) == (dest_err is None)
pending = {}
pending_ex = None
try:
fds = tuple([x for x in (src_out, src_err) if x is not None])
while fds:
ready_fds, _, _ = select.select(fds, [], [])
width = tty_width()
for fd in ready_fds:
buf = os.read(fd, 4096)
dest = dest_out if fd == src_out else dest_err
if not buf:
fds = tuple([x for x in fds if x is not fd])
print_clean_line(dest, pending.pop(fd, []), width)
else:
split = sep_rx.split(buf)
if len(split) > 2:
while len(split) > 1:
content, sep = split[:2]
split = split[2:]
print_clean_line(dest,
pending.pop(fd, []) + [content],
width,
sep)
else:
assert(len(split) == 1)
pending.setdefault(fd, []).extend(split)
except BaseException as ex:
pending_ex = chain_ex(add_ex_tb(ex), pending_ex)
try:
# Try to finish each of the streams
for fd, pending_items in compat.items(pending):
dest = dest_out if fd == src_out else dest_err
try:
print_clean_line(dest, pending_items, width)
except (EnvironmentError, EOFError) as ex:
pending_ex = chain_ex(add_ex_tb(ex), pending_ex)
except BaseException as ex:
pending_ex = chain_ex(add_ex_tb(ex), pending_ex)
if pending_ex:
raise pending_ex
def filter_output(src_out, src_err, dest_out, dest_err):
"""Transfer data from src_out to dest_out and src_err to dest_err via
print_clean_line until src_out and src_err close."""
global sep_rx
assert not isinstance(src_out, bool)
assert not isinstance(src_err, bool)
assert not isinstance(dest_out, bool)
assert not isinstance(dest_err, bool)
assert src_out is not None or src_err is not None
assert (src_out is None) == (dest_out is None)
assert (src_err is None) == (dest_err is None)
pending = {}
pending_ex = None
try:
fds = tuple([x for x in (src_out, src_err) if x is not None])
while fds:
ready_fds, _, _ = select.select(fds, [], [])
width = tty_width()
for fd in ready_fds:
buf = os.read(fd, 4096)
dest = dest_out if fd == src_out else dest_err
if not buf:
fds = tuple([x for x in fds if x is not fd])
print_clean_line(dest, pending.pop(fd, []), width)
else:
split = sep_rx.split(buf)
if len(split) > 2:
while len(split) > 1:
content, sep = split[:2]
split = split[2:]
print_clean_line(dest,
pending.pop(fd, []) + [content],
width, sep)
else:
assert (len(split) == 1)
pending.setdefault(fd, []).extend(split)
except BaseException as ex:
pending_ex = chain_ex(add_ex_tb(ex), pending_ex)
try:
# Try to finish each of the streams
for fd, pending_items in compat.items(pending):
dest = dest_out if fd == src_out else dest_err
try:
print_clean_line(dest, pending_items, width)
except (EnvironmentError, EOFError) as ex:
pending_ex = chain_ex(add_ex_tb(ex), pending_ex)
except BaseException as ex:
pending_ex = chain_ex(add_ex_tb(ex), pending_ex)
if pending_ex:
raise pending_ex
from bup import _helpers, compat, hashsplit, path, midx, bloom, xstat
from bup.compat import (buffer, byte_int, bytes_from_byte, bytes_from_uint,
environ, items, range, reraise)
from bup.io import path_msg
from bup.helpers import (Sha1, add_error, chunkyreader, debug1, debug2, exo,
fdatasync, hostname, localtime, log, merge_dict,
merge_iter, mmap_read, mmap_readwrite, parse_num,
progress, qprogress, stat_if_exists, unlink,
utc_offset_str)
from bup.pwdgrp import username, userfullname
verbose = 0
repodir = None # The default repository, once initialized
_typemap = {b'blob': 3, b'tree': 2, b'commit': 1, b'tag': 4}
_typermap = {v: k for k, v in items(_typemap)}
_total_searches = 0
_total_steps = 0
class GitError(Exception):
pass
def _gitenv(repo_dir=None):
if not repo_dir:
repo_dir = repo()
return merge_dict(environ, {b'GIT_DIR': os.path.abspath(repo_dir)})
def main():
handle_ctrl_c()
is_reverse = environ.get(b'BUP_SERVER_REVERSE')
opt = parse_args(compat.argv)
git.check_repo_or_die()
if opt.source:
opt.source = argv_bytes(opt.source)
if opt.bwlimit:
client.bwlimit = parse_num(opt.bwlimit)
if is_reverse and opt.remote:
misuse("don't use -r in reverse mode; it's automatic")
if opt.remote:
opt.remote = argv_bytes(opt.remote)
if opt.remote or is_reverse:
dest_repo = RemoteRepo(opt.remote)
else:
dest_repo = LocalRepo()
with dest_repo as dest_repo:
with LocalRepo(repo_dir=opt.source) as src_repo:
with dest_repo.new_packwriter(compression_level=opt.compress) as writer:
# Resolve and validate all sources and destinations,
# implicit or explicit, and do it up-front, so we can
# fail before we start writing (for any obviously
# broken cases).
target_items = resolve_targets(opt.target_specs,
src_repo, dest_repo)
updated_refs = {} # ref_name -> (original_ref, tip_commit(bin))
no_ref_info = (None, None)
handlers = {'ff': handle_ff,
'append': handle_append,
'force-pick': handle_pick,
'pick': handle_pick,
'new-tag': handle_new_tag,
'replace': handle_replace,
'unnamed': handle_unnamed}
for item in target_items:
debug1('get-spec: %r\n' % (item.spec,))
debug1('get-src: %s\n' % loc_desc(item.src))
debug1('get-dest: %s\n' % loc_desc(item.dest))
dest_path = item.dest and item.dest.path
if dest_path:
if dest_path.startswith(b'/.tag/'):
dest_ref = b'refs/tags/%s' % dest_path[6:]
else:
dest_ref = b'refs/heads/%s' % dest_path[1:]
else:
dest_ref = None
dest_hash = item.dest and item.dest.hash
orig_ref, cur_ref = updated_refs.get(dest_ref, no_ref_info)
orig_ref = orig_ref or dest_hash
cur_ref = cur_ref or dest_hash
handler = handlers[item.spec.method]
item_result = handler(item, src_repo, writer, opt)
if len(item_result) > 1:
new_id, tree = item_result
else:
new_id = item_result[0]
if not dest_ref:
log_item(item.spec.src, item.src.type, opt)
else:
updated_refs[dest_ref] = (orig_ref, new_id)
if dest_ref.startswith(b'refs/tags/'):
log_item(item.spec.src, item.src.type, opt, tag=new_id)
else:
log_item(item.spec.src, item.src.type, opt,
tree=tree, commit=new_id)
# Only update the refs at the very end, once the writer is
# closed, so that if something goes wrong above, the old refs
# will be undisturbed.
for ref_name, info in items(updated_refs):
orig_ref, new_ref = info
try:
dest_repo.update_ref(ref_name, new_ref, orig_ref)
if opt.verbose:
new_hex = hexlify(new_ref)
if orig_ref:
orig_hex = hexlify(orig_ref)
log('updated %r (%s -> %s)\n' % (ref_name, orig_hex, new_hex))
else:
log('updated %r (%s)\n' % (ref_name, new_hex))
except (git.GitError, client.ClientError) as ex:
add_error('unable to update ref %r: %s' % (ref_name, ex))
if saved_errors:
log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))
sys.exit(1)
def _test_replace(get_disposition, src_info):
print('blarg:', repr(src_info), file=sys.stderr)
wvstart(get_disposition + ' --replace to root fails')
for item in (b'.tag/tinyfile', b'src/latest' + src_info['tinyfile-path'],
b'.tag/subtree', b'src/latest' + src_info['subtree-vfs-path'],
b'.tag/commit-1', b'src/latest', b'src'):
exr = run_get(get_disposition, b'--replace', (item, b'/'))
wvpassne(0, exr.rc)
verify_rx(br'impossible; can only overwrite branch or tag', exr.err)
tinyfile_id = src_info['tinyfile-id']
tinyfile_path = src_info['tinyfile-path']
subtree_vfs_path = src_info['subtree-vfs-path']
subtree_id = src_info['subtree-id']
commit_2_id = src_info['commit-2-id']
tree_2_id = src_info['tree-2-id']
# Anything to tag
existing_items = {
'nothing': None,
'blob': (b'.tag/tinyfile', b'.tag/obj'),
'tree': (b'.tag/tree-1', b'.tag/obj'),
'commit': (b'.tag/commit-1', b'.tag/obj')
}
for ex_type, ex_ref in items(existing_items):
wvstart(get_disposition + ' --replace ' + ex_type + ' with blob tag')
for item in (b'.tag/tinyfile', b'src/latest' + tinyfile_path):
exr = run_get(get_disposition,
b'--replace', (item, b'.tag/obj'),
given=ex_ref)
wvpasseq(0, exr.rc)
validate_blob(tinyfile_id, tinyfile_id)
verify_only_refs(heads=[], tags=(b'obj', ))
wvstart(get_disposition + ' --replace ' + ex_type + ' with tree tag')
for item in (b'.tag/subtree', b'src/latest' + subtree_vfs_path):
exr = run_get(get_disposition,
b'--replace', (item, b'.tag/obj'),
given=ex_ref)
validate_tree(subtree_id, subtree_id)
verify_only_refs(heads=[], tags=(b'obj', ))
wvstart(get_disposition + ' --replace ' + ex_type +
' with commitish tag')
for item in (b'.tag/commit-2', b'src/latest', b'src'):
exr = run_get(get_disposition,
b'--replace', (item, b'.tag/obj'),
given=ex_ref)
validate_tagged_save(b'obj',
getcwd() + b'/src', commit_2_id, tree_2_id,
b'src-2', exr.out)
verify_only_refs(heads=[], tags=(b'obj', ))
# Committish to branch.
existing_items = (('nothing', None), ('branch', (b'.tag/commit-1',
b'obj')))
for ex_type, ex_ref in existing_items:
for item_type, item in (('commit', b'.tag/commit-2'),
('save', b'src/latest'), ('branch',
b'src')):
wvstart(get_disposition + ' --replace ' + ex_type + ' with ' +
item_type)
exr = run_get(get_disposition,
b'--replace', (item, b'obj'),
given=ex_ref)
validate_save(b'obj/latest',
getcwd() + b'/src', commit_2_id, tree_2_id,
b'src-2', exr.out)
verify_only_refs(heads=(b'obj', ), tags=[])
# Not committish to branch
existing_items = (('nothing', None), ('branch', (b'.tag/commit-1',
b'obj')))
for ex_type, ex_ref in existing_items:
for item_type, item in (('blob', b'.tag/tinyfile'),
('blob', b'src/latest' + tinyfile_path),
('tree', b'.tag/subtree'),
('tree',
b'src/latest' + subtree_vfs_path)):
wvstart(get_disposition + ' --replace branch with ' +
item_type + ' given ' + ex_type + ' fails')
exr = run_get(get_disposition,
b'--replace', (item, b'obj'),
given=ex_ref)
wvpassne(0, exr.rc)
verify_rx(br'cannot overwrite branch with .+ for', exr.err)
wvstart(get_disposition + ' --replace, implicit destinations')
exr = run_get(get_disposition, b'--replace', b'src')
validate_save(b'src/latest',
getcwd() + b'/src', commit_2_id, tree_2_id, b'src-2',
exr.out)
verify_only_refs(heads=(b'src', ), tags=[])
exr = run_get(get_disposition, b'--replace', b'.tag/commit-2')
validate_tagged_save(b'commit-2',
getcwd() + b'/src', commit_2_id, tree_2_id,
b'src-2', exr.out)
verify_only_refs(heads=[], tags=(b'commit-2', ))
