def test_cache_behavior(): orig_max = vfs._cache_max_items try: vfs._cache_max_items = 2 vfs.clear_cache() wvpasseq({}, vfs._cache) wvpasseq([], vfs._cache_keys) wvfail(vfs._cache_keys) wvexcept(Exception, vfs.cache_notice, b'x', 1) key_0 = b'itm:' + b'\0' * 20 key_1 = b'itm:' + b'\1' * 20 key_2 = b'itm:' + b'\2' * 20 vfs.cache_notice(key_0, b'something') wvpasseq({key_0: b'something'}, vfs._cache) wvpasseq([key_0], vfs._cache_keys) vfs.cache_notice(key_1, b'something else') wvpasseq({key_0: b'something', key_1: b'something else'}, vfs._cache) wvpasseq(frozenset([key_0, key_1]), frozenset(vfs._cache_keys)) vfs.cache_notice(key_2, b'and also') wvpasseq(2, len(vfs._cache)) wvpass( frozenset(items(vfs._cache)) < frozenset( items({ key_0: b'something', key_1: b'something else', key_2: b'and also' }))) wvpasseq(2, len(vfs._cache_keys)) wvpass(frozenset(vfs._cache_keys) < frozenset([key_0, key_1, key_2])) vfs.clear_cache() wvpasseq({}, vfs._cache) wvpasseq([], vfs._cache_keys) finally: vfs._cache_max_items = orig_max vfs.clear_cache()
def bup_rm(repo, paths, compression=6, verbosity=None): dead_branches, dead_saves = dead_items(repo, paths) die_if_errors('not proceeding with any removals\n') updated_refs = {} # ref_name -> (original_ref, tip_commit(bin)) for branchname, branchitem in compat.items(dead_branches): ref = 'refs/heads/' + branchname assert (not ref in updated_refs) updated_refs[ref] = (branchitem.oid, None) if dead_saves: writer = git.PackWriter(compression_level=compression) try: for branch, saves in compat.items(dead_saves): assert (saves) updated_refs['refs/heads/' + branch] = rm_saves(saves, writer) except: if writer: writer.abort() raise else: if writer: # Must close before we can update the ref(s) below. writer.close() # Only update the refs here, at the very end, so that if something # goes wrong above, the old refs will be undisturbed. Make an attempt # to update each ref. for ref_name, info in compat.items(updated_refs): orig_ref, new_ref = info try: if not new_ref: git.delete_ref(ref_name, orig_ref.encode('hex')) else: git.update_ref(ref_name, new_ref, orig_ref) if verbosity: new_hex = new_ref.encode('hex') if orig_ref: orig_hex = orig_ref.encode('hex') log('updated %r (%s -> %s)\n' % (ref_name, orig_hex, new_hex)) else: log('updated %r (%s)\n' % (ref_name, new_hex)) except (git.GitError, ClientError) as ex: if new_ref: add_error('while trying to update %r (%s -> %s): %s' % (ref_name, orig_ref, new_ref, ex)) else: add_error('while trying to delete %r (%s): %s' % (ref_name, orig_ref, ex))
def bup_rm(repo, paths, compression=6, verbosity=None): dead_branches, dead_saves = dead_items(repo, paths) die_if_errors('not proceeding with any removals\n') updated_refs = {} # ref_name -> (original_ref, tip_commit(bin)) for branchname, branchitem in compat.items(dead_branches): ref = 'refs/heads/' + branchname assert(not ref in updated_refs) updated_refs[ref] = (branchitem.oid, None) if dead_saves: writer = git.PackWriter(compression_level=compression) try: for branch, saves in compat.items(dead_saves): assert(saves) updated_refs['refs/heads/' + branch] = rm_saves(saves, writer) except: if writer: writer.abort() raise else: if writer: # Must close before we can update the ref(s) below. writer.close() # Only update the refs here, at the very end, so that if something # goes wrong above, the old refs will be undisturbed. Make an attempt # to update each ref. for ref_name, info in compat.items(updated_refs): orig_ref, new_ref = info try: if not new_ref: git.delete_ref(ref_name, orig_ref.encode('hex')) else: git.update_ref(ref_name, new_ref, orig_ref) if verbosity: new_hex = new_ref.encode('hex') if orig_ref: orig_hex = orig_ref.encode('hex') log('updated %r (%s -> %s)\n' % (ref_name, orig_hex, new_hex)) else: log('updated %r (%s)\n' % (ref_name, new_hex)) except (git.GitError, ClientError) as ex: if new_ref: add_error('while trying to update %r (%s -> %s): %s' % (ref_name, orig_ref, new_ref, ex)) else: add_error('while trying to delete %r (%s): %s' % (ref_name, orig_ref, ex))
def _argmax_base(command): base_size = 2048 for c in command: base_size += len(command) + 1 for k, v in compat.items(environ): base_size += len(k) + len(v) + 2 + sizeof(c_void_p) return base_size
def expected_retentions(utcs, utc_start, spec): if not spec: return utcs utcs = sorted(utcs, reverse=True) period_start = dict(spec) for kind, duration in compat.items(period_start): period_start[kind] = utc_start - period_as_secs(duration) period_start = defaultdict(lambda: float('inf'), period_start) all = list(takewhile(lambda x: x >= period_start[b'all'], utcs)) utcs = list(dropwhile(lambda x: x >= period_start[b'all'], utcs)) matches = takewhile(lambda x: x >= period_start[b'dailies'], utcs) dailies = [max(day_utcs) for yday, day_utcs in groupby(matches, lambda x: localtime(x).tm_yday)] utcs = list(dropwhile(lambda x: x >= period_start[b'dailies'], utcs)) matches = takewhile(lambda x: x >= period_start[b'monthlies'], utcs) monthlies = [max(month_utcs) for month, month_utcs in groupby(matches, lambda x: localtime(x).tm_mon)] utcs = dropwhile(lambda x: x >= period_start[b'monthlies'], utcs) matches = takewhile(lambda x: x >= period_start[b'yearlies'], utcs) yearlies = [max(year_utcs) for year, year_utcs in groupby(matches, lambda x: localtime(x).tm_year)] return chain(all, dailies, monthlies, yearlies)
def __init__(self, filename): # Map a "dev:ino" node to a list of paths associated with that node. self._node_paths = {} # Map a path to a "dev:ino" node. self._path_node = {} self._filename = filename self._save_prepared = None self._tmpname = None f = None try: f = open(filename, 'rb') except IOError as e: if e.errno == errno.ENOENT: pass else: raise if f: try: self._node_paths = pickle_load(f) finally: f.close() f = None # Set up the reverse hard link index. for node, paths in compat.items(self._node_paths): for path in paths: self._path_node[path] = node
def __init__(self, filename): # Map a "dev:ino" node to a list of paths associated with that node. self._node_paths = {} # Map a path to a "dev:ino" node. self._path_node = {} self._filename = filename self._save_prepared = None self._tmpname = None f = None try: f = open(filename, 'r') except IOError as e: if e.errno == errno.ENOENT: pass else: raise if f: try: self._node_paths = cPickle.load(f) finally: f.close() f = None # Set up the reverse hard link index. for node, paths in compat.items(self._node_paths): for path in paths: self._path_node[path] = node
def verify_only_refs(**kwargs): for kind, refs in items(kwargs): if kind == 'heads': abs_refs = [b'refs/heads/' + ref for ref in refs] karg = b'--heads' elif kind == 'tags': abs_refs = [b'refs/tags/' + ref for ref in refs] karg = b'--tags' else: raise TypeError('unexpected keyword argument %r' % kind) if abs_refs: verify_rcz([ b'git', b'--git-dir', b'get-dest', b'show-ref', b'--verify', karg ] + abs_refs) exr = exo((b'git', b'--git-dir', b'get-dest', b'show-ref', karg), check=False) wvpasseq(0, exr.rc) expected_refs = sorted(abs_refs) repo_refs = sorted([x.split()[1] for x in exr.out.splitlines()]) wvpasseq(expected_refs, repo_refs) else: # FIXME: can we just check "git show-ref --heads == ''"? exr = exo((b'git', b'--git-dir', b'get-dest', b'show-ref', karg), check=False) wvpasseq(1, exr.rc) wvpasseq(b'', exr.out.strip())
def bup_rm(repo, paths, verbosity=None): dead_branches, dead_saves = dead_items(repo, paths) die_if_errors('not proceeding with any removals\n') updated_refs = {} # ref_name -> (original_ref, tip_commit(bin)) for branchname, branchitem in compat.items(dead_branches): ref = b'refs/heads/' + branchname assert (not ref in updated_refs) updated_refs[ref] = (branchitem.oid, None) if dead_saves: try: for branch, saves in compat.items(dead_saves): assert (saves) updated_refs[b'refs/heads/' + branch] = rm_saves(repo, saves) except BaseException as ex: with pending_raise(ex): repo.abort_writing() # Only update the refs here, at the very end, so that if something # goes wrong above, the old refs will be undisturbed. Make an attempt # to update each ref. for ref_name, info in compat.items(updated_refs): orig_ref, new_ref = info try: if not new_ref: repo.delete_ref(ref_name, orig_ref) else: repo.update_ref(ref_name, new_ref, orig_ref) if verbosity: log('updated %s (%s%s)\n' % (path_msg(ref_name), hexstr(orig_ref) + ' -> ' if orig_ref else '', hexstr(new_ref))) except (git.GitError, ClientError) as ex: if new_ref: add_error('while trying to update %s (%s%s): %s' % (path_msg(ref_name), hexstr(orig_ref) + ' -> ' if orig_ref else '', hexstr(new_ref), ex)) else: add_error('while trying to delete %r (%s): %s' % (ref_name, hexstr(orig_ref), ex))
def filter_output(srcs, dests): """Transfer data from file descriptors in srcs to the corresponding file descriptors in dests print_clean_line until all of the srcs have closed. """ global sep_rx assert all(type(x) in int_types for x in srcs) assert all(type(x) in int_types for x in srcs) assert len(srcs) == len(dests) srcs = tuple(srcs) dest_for = dict(zip(srcs, dests)) pending = {} pending_ex = None try: while srcs: ready_fds, _, _ = select.select(srcs, [], []) width = tty_width() for fd in ready_fds: buf = os.read(fd, 4096) dest = dest_for[fd] if not buf: srcs = tuple([x for x in srcs if x is not fd]) print_clean_line(dest, pending.pop(fd, []), width) else: split = sep_rx.split(buf) while len(split) > 1: content, sep = split[:2] split = split[2:] print_clean_line(dest, pending.pop(fd, []) + [content], width, sep) assert len(split) == 1 if split[0]: pending.setdefault(fd, []).extend(split) except BaseException as ex: pending_ex = add_ex_ctx(add_ex_tb(ex), pending_ex) try: # Try to finish each of the streams for fd, pending_items in compat.items(pending): dest = dest_for[fd] width = tty_width() try: print_clean_line(dest, pending_items, width) except (EnvironmentError, EOFError) as ex: pending_ex = add_ex_ctx(add_ex_tb(ex), pending_ex) except BaseException as ex: pending_ex = add_ex_ctx(add_ex_tb(ex), pending_ex) if pending_ex: raise pending_ex
def filter_output(src_out, src_err, dest_out, dest_err): """Transfer data from src_out to dest_out and src_err to dest_err via print_clean_line until src_out and src_err close.""" global sep_rx assert not isinstance(src_out, bool) assert not isinstance(src_err, bool) assert not isinstance(dest_out, bool) assert not isinstance(dest_err, bool) assert src_out is not None or src_err is not None assert (src_out is None) == (dest_out is None) assert (src_err is None) == (dest_err is None) pending = {} pending_ex = None try: fds = tuple([x for x in (src_out, src_err) if x is not None]) while fds: ready_fds, _, _ = select.select(fds, [], []) width = tty_width() for fd in ready_fds: buf = os.read(fd, 4096) dest = dest_out if fd == src_out else dest_err if not buf: fds = tuple([x for x in fds if x is not fd]) print_clean_line(dest, pending.pop(fd, []), width) else: split = sep_rx.split(buf) if len(split) > 2: while len(split) > 1: content, sep = split[:2] split = split[2:] print_clean_line(dest, pending.pop(fd, []) + [content], width, sep) else: assert(len(split) == 1) pending.setdefault(fd, []).extend(split) except BaseException as ex: pending_ex = chain_ex(add_ex_tb(ex), pending_ex) try: # Try to finish each of the streams for fd, pending_items in compat.items(pending): dest = dest_out if fd == src_out else dest_err try: print_clean_line(dest, pending_items, width) except (EnvironmentError, EOFError) as ex: pending_ex = chain_ex(add_ex_tb(ex), pending_ex) except BaseException as ex: pending_ex = chain_ex(add_ex_tb(ex), pending_ex) if pending_ex: raise pending_ex
def filter_output(src_out, src_err, dest_out, dest_err): """Transfer data from src_out to dest_out and src_err to dest_err via print_clean_line until src_out and src_err close.""" global sep_rx assert not isinstance(src_out, bool) assert not isinstance(src_err, bool) assert not isinstance(dest_out, bool) assert not isinstance(dest_err, bool) assert src_out is not None or src_err is not None assert (src_out is None) == (dest_out is None) assert (src_err is None) == (dest_err is None) pending = {} pending_ex = None try: fds = tuple([x for x in (src_out, src_err) if x is not None]) while fds: ready_fds, _, _ = select.select(fds, [], []) width = tty_width() for fd in ready_fds: buf = os.read(fd, 4096) dest = dest_out if fd == src_out else dest_err if not buf: fds = tuple([x for x in fds if x is not fd]) print_clean_line(dest, pending.pop(fd, []), width) else: split = sep_rx.split(buf) if len(split) > 2: while len(split) > 1: content, sep = split[:2] split = split[2:] print_clean_line(dest, pending.pop(fd, []) + [content], width, sep) else: assert (len(split) == 1) pending.setdefault(fd, []).extend(split) except BaseException as ex: pending_ex = chain_ex(add_ex_tb(ex), pending_ex) try: # Try to finish each of the streams for fd, pending_items in compat.items(pending): dest = dest_out if fd == src_out else dest_err try: print_clean_line(dest, pending_items, width) except (EnvironmentError, EOFError) as ex: pending_ex = chain_ex(add_ex_tb(ex), pending_ex) except BaseException as ex: pending_ex = chain_ex(add_ex_tb(ex), pending_ex) if pending_ex: raise pending_ex
from bup import _helpers, compat, hashsplit, path, midx, bloom, xstat from bup.compat import (buffer, byte_int, bytes_from_byte, bytes_from_uint, environ, items, range, reraise) from bup.io import path_msg from bup.helpers import (Sha1, add_error, chunkyreader, debug1, debug2, exo, fdatasync, hostname, localtime, log, merge_dict, merge_iter, mmap_read, mmap_readwrite, parse_num, progress, qprogress, stat_if_exists, unlink, utc_offset_str) from bup.pwdgrp import username, userfullname verbose = 0 repodir = None # The default repository, once initialized _typemap = {b'blob': 3, b'tree': 2, b'commit': 1, b'tag': 4} _typermap = {v: k for k, v in items(_typemap)} _total_searches = 0 _total_steps = 0 class GitError(Exception): pass def _gitenv(repo_dir=None): if not repo_dir: repo_dir = repo() return merge_dict(environ, {b'GIT_DIR': os.path.abspath(repo_dir)})
def main(): handle_ctrl_c() is_reverse = environ.get(b'BUP_SERVER_REVERSE') opt = parse_args(compat.argv) git.check_repo_or_die() if opt.source: opt.source = argv_bytes(opt.source) if opt.bwlimit: client.bwlimit = parse_num(opt.bwlimit) if is_reverse and opt.remote: misuse("don't use -r in reverse mode; it's automatic") if opt.remote: opt.remote = argv_bytes(opt.remote) if opt.remote or is_reverse: dest_repo = RemoteRepo(opt.remote) else: dest_repo = LocalRepo() with dest_repo as dest_repo: with LocalRepo(repo_dir=opt.source) as src_repo: with dest_repo.new_packwriter(compression_level=opt.compress) as writer: # Resolve and validate all sources and destinations, # implicit or explicit, and do it up-front, so we can # fail before we start writing (for any obviously # broken cases). target_items = resolve_targets(opt.target_specs, src_repo, dest_repo) updated_refs = {} # ref_name -> (original_ref, tip_commit(bin)) no_ref_info = (None, None) handlers = {'ff': handle_ff, 'append': handle_append, 'force-pick': handle_pick, 'pick': handle_pick, 'new-tag': handle_new_tag, 'replace': handle_replace, 'unnamed': handle_unnamed} for item in target_items: debug1('get-spec: %r\n' % (item.spec,)) debug1('get-src: %s\n' % loc_desc(item.src)) debug1('get-dest: %s\n' % loc_desc(item.dest)) dest_path = item.dest and item.dest.path if dest_path: if dest_path.startswith(b'/.tag/'): dest_ref = b'refs/tags/%s' % dest_path[6:] else: dest_ref = b'refs/heads/%s' % dest_path[1:] else: dest_ref = None dest_hash = item.dest and item.dest.hash orig_ref, cur_ref = updated_refs.get(dest_ref, no_ref_info) orig_ref = orig_ref or dest_hash cur_ref = cur_ref or dest_hash handler = handlers[item.spec.method] item_result = handler(item, src_repo, writer, opt) if len(item_result) > 1: new_id, tree = item_result else: new_id = item_result[0] if not dest_ref: log_item(item.spec.src, item.src.type, opt) else: updated_refs[dest_ref] = (orig_ref, new_id) if dest_ref.startswith(b'refs/tags/'): log_item(item.spec.src, item.src.type, opt, tag=new_id) else: log_item(item.spec.src, item.src.type, opt, tree=tree, commit=new_id) # Only update the refs at the very end, once the writer is # closed, so that if something goes wrong above, the old refs # will be undisturbed. for ref_name, info in items(updated_refs): orig_ref, new_ref = info try: dest_repo.update_ref(ref_name, new_ref, orig_ref) if opt.verbose: new_hex = hexlify(new_ref) if orig_ref: orig_hex = hexlify(orig_ref) log('updated %r (%s -> %s)\n' % (ref_name, orig_hex, new_hex)) else: log('updated %r (%s)\n' % (ref_name, new_hex)) except (git.GitError, client.ClientError) as ex: add_error('unable to update ref %r: %s' % (ref_name, ex)) if saved_errors: log('WARNING: %d errors encountered while saving.\n' % len(saved_errors)) sys.exit(1)
def _test_replace(get_disposition, src_info): print('blarg:', repr(src_info), file=sys.stderr) wvstart(get_disposition + ' --replace to root fails') for item in (b'.tag/tinyfile', b'src/latest' + src_info['tinyfile-path'], b'.tag/subtree', b'src/latest' + src_info['subtree-vfs-path'], b'.tag/commit-1', b'src/latest', b'src'): exr = run_get(get_disposition, b'--replace', (item, b'/')) wvpassne(0, exr.rc) verify_rx(br'impossible; can only overwrite branch or tag', exr.err) tinyfile_id = src_info['tinyfile-id'] tinyfile_path = src_info['tinyfile-path'] subtree_vfs_path = src_info['subtree-vfs-path'] subtree_id = src_info['subtree-id'] commit_2_id = src_info['commit-2-id'] tree_2_id = src_info['tree-2-id'] # Anything to tag existing_items = { 'nothing': None, 'blob': (b'.tag/tinyfile', b'.tag/obj'), 'tree': (b'.tag/tree-1', b'.tag/obj'), 'commit': (b'.tag/commit-1', b'.tag/obj') } for ex_type, ex_ref in items(existing_items): wvstart(get_disposition + ' --replace ' + ex_type + ' with blob tag') for item in (b'.tag/tinyfile', b'src/latest' + tinyfile_path): exr = run_get(get_disposition, b'--replace', (item, b'.tag/obj'), given=ex_ref) wvpasseq(0, exr.rc) validate_blob(tinyfile_id, tinyfile_id) verify_only_refs(heads=[], tags=(b'obj', )) wvstart(get_disposition + ' --replace ' + ex_type + ' with tree tag') for item in (b'.tag/subtree', b'src/latest' + subtree_vfs_path): exr = run_get(get_disposition, b'--replace', (item, b'.tag/obj'), given=ex_ref) validate_tree(subtree_id, subtree_id) verify_only_refs(heads=[], tags=(b'obj', )) wvstart(get_disposition + ' --replace ' + ex_type + ' with commitish tag') for item in (b'.tag/commit-2', b'src/latest', b'src'): exr = run_get(get_disposition, b'--replace', (item, b'.tag/obj'), given=ex_ref) validate_tagged_save(b'obj', getcwd() + b'/src', commit_2_id, tree_2_id, b'src-2', exr.out) verify_only_refs(heads=[], tags=(b'obj', )) # Committish to branch. existing_items = (('nothing', None), ('branch', (b'.tag/commit-1', b'obj'))) for ex_type, ex_ref in existing_items: for item_type, item in (('commit', b'.tag/commit-2'), ('save', b'src/latest'), ('branch', b'src')): wvstart(get_disposition + ' --replace ' + ex_type + ' with ' + item_type) exr = run_get(get_disposition, b'--replace', (item, b'obj'), given=ex_ref) validate_save(b'obj/latest', getcwd() + b'/src', commit_2_id, tree_2_id, b'src-2', exr.out) verify_only_refs(heads=(b'obj', ), tags=[]) # Not committish to branch existing_items = (('nothing', None), ('branch', (b'.tag/commit-1', b'obj'))) for ex_type, ex_ref in existing_items: for item_type, item in (('blob', b'.tag/tinyfile'), ('blob', b'src/latest' + tinyfile_path), ('tree', b'.tag/subtree'), ('tree', b'src/latest' + subtree_vfs_path)): wvstart(get_disposition + ' --replace branch with ' + item_type + ' given ' + ex_type + ' fails') exr = run_get(get_disposition, b'--replace', (item, b'obj'), given=ex_ref) wvpassne(0, exr.rc) verify_rx(br'cannot overwrite branch with .+ for', exr.err) wvstart(get_disposition + ' --replace, implicit destinations') exr = run_get(get_disposition, b'--replace', b'src') validate_save(b'src/latest', getcwd() + b'/src', commit_2_id, tree_2_id, b'src-2', exr.out) verify_only_refs(heads=(b'src', ), tags=[]) exr = run_get(get_disposition, b'--replace', b'.tag/commit-2') validate_tagged_save(b'commit-2', getcwd() + b'/src', commit_2_id, tree_2_id, b'src-2', exr.out) verify_only_refs(heads=[], tags=(b'commit-2', ))