def test_metadata_method(): with no_lingering_errors(), test_tempdir('bup-tmetadata-') as tmpdir: bup_dir = tmpdir + '/bup' data_path = tmpdir + '/foo' os.mkdir(data_path) ex('touch', data_path + '/file') ex('ln', '-s', 'file', data_path + '/symlink') test_time1 = 13 * 1000000000 test_time2 = 42 * 1000000000 utime(data_path + '/file', (0, test_time1)) lutime(data_path + '/symlink', (0, 0)) utime(data_path, (0, test_time2)) ex(bup_path, '-d', bup_dir, 'init') ex(bup_path, '-d', bup_dir, 'index', '-v', data_path) ex(bup_path, '-d', bup_dir, 'save', '-tvvn', 'test', data_path) git.check_repo_or_die(bup_dir) top = vfs.RefList(None) n = top.lresolve('/test/latest' + resolve_parent(data_path)) m = n.metadata() WVPASS(m.mtime == test_time2) WVPASS(len(n.subs()) == 2) WVPASS(n.name == 'foo') WVPASS(set([x.name for x in n.subs()]) == set(['file', 'symlink'])) for sub in n: if sub.name == 'file': m = sub.metadata() WVPASS(m.mtime == test_time1) elif sub.name == 'symlink': m = sub.metadata() WVPASS(m.mtime == 0)
def test_metadata_method(): tmpdir = tempfile.mkdtemp(prefix='bup-tmetadata-') try: bup_dir = tmpdir + '/bup' data_path = tmpdir + '/foo' os.mkdir(data_path) ex('touch', data_path + '/file') ex('ln', '-s', 'file', data_path + '/symlink') test_time1 = 13 * 1000000000 test_time2 = 42 * 1000000000 utime(data_path + '/file', (0, test_time1)) lutime(data_path + '/symlink', (0, 0)) utime(data_path, (0, test_time2)) ex(bup_path, '-d', bup_dir, 'init') ex(bup_path, '-d', bup_dir, 'index', '-v', data_path) ex(bup_path, '-d', bup_dir, 'save', '-tvvn', 'test', data_path) git.check_repo_or_die(bup_dir) top = vfs.RefList(None) n = top.lresolve('/test/latest' + realpath(data_path)) m = n.metadata() WVPASS(m.mtime == test_time2) WVPASS(len(n.subs()) == 2) WVPASS(n.name == 'foo') WVPASS(set([x.name for x in n.subs()]) == set(['file', 'symlink'])) for sub in n: if sub.name == 'file': m = sub.metadata() WVPASS(m.mtime == test_time1) elif sub.name == 'symlink': m = sub.metadata() WVPASS(m.mtime == 0) finally: subprocess.call(['rm', '-rf', tmpdir])
def call_save(dirname, optDict, callbacks={}): global count, subcount, lastremain opt = OptionsDict(optDict) flags = {} extra = [dirname] git.check_repo_or_die() if not (opt.tree or opt.commit or opt.name): o.fatal("use one or more of -t, -c, -n") if not extra: o.fatal("no filenames given") opt.progress = (istty2 and not opt.quiet) opt.smaller = parse_num(opt.smaller or 0) if opt.bwlimit: client.bwlimit = parse_num(opt.bwlimit) if opt.date: date = parse_date_or_fatal(opt.date, o.fatal) else: date = time.time() if opt.strip and opt.strip_path: o.fatal("--strip is incompatible with --strip-path") graft_points = [] if opt.graft: if opt.strip: o.fatal("--strip is incompatible with --graft") if opt.strip_path: o.fatal("--strip-path is incompatible with --graft") for (option, parameter) in flags: if option == "--graft": splitted_parameter = parameter.split('=') if len(splitted_parameter) != 2: o.fatal("a graft point must be of the form old_path=new_path") old_path, new_path = splitted_parameter if not (old_path and new_path): o.fatal("a graft point cannot be empty") graft_points.append((realpath(old_path), realpath(new_path))) is_reverse = os.environ.get('BUP_SERVER_REVERSE') if is_reverse and opt.remote: o.fatal("don't use -r in reverse mode; it's automatic") if opt.name and opt.name.startswith('.'): o.fatal("'%s' is not a valid branch name" % opt.name) refname = opt.name and 'refs/heads/%s' % opt.name or None if opt.remote or is_reverse: try: cli = client.Client(opt.remote) except client.ClientError, e: log('error: %s' % e) sys.exit(1) oldref = refname and cli.read_ref(refname) or None w = cli.new_packwriter(compression_level=opt.compress)
def test_metadata_method(): tmpdir = tempfile.mkdtemp(prefix="bup-tmetadata-") try: bup_dir = tmpdir + "/bup" data_path = tmpdir + "/foo" os.mkdir(data_path) ex("touch", data_path + "/file") ex("ln", "-s", "file", data_path + "/symlink") test_time1 = 13 * 1000000000 test_time2 = 42 * 1000000000 utime(data_path + "/file", (0, test_time1)) lutime(data_path + "/symlink", (0, 0)) utime(data_path, (0, test_time2)) ex(bup_path, "-d", bup_dir, "init") ex(bup_path, "-d", bup_dir, "index", "-v", data_path) ex(bup_path, "-d", bup_dir, "save", "-tvvn", "test", data_path) git.check_repo_or_die(bup_dir) top = vfs.RefList(None) n = top.lresolve("/test/latest" + realpath(data_path)) m = n.metadata() WVPASS(m.mtime == test_time2) WVPASS(len(n.subs()) == 2) WVPASS(n.name == "foo") WVPASS(set([x.name for x in n.subs()]) == set(["file", "symlink"])) for sub in n: if sub.name == "file": m = sub.metadata() WVPASS(m.mtime == test_time1) elif sub.name == "symlink": m = sub.metadata() WVPASS(m.mtime == 0) finally: subprocess.call(["rm", "-rf", tmpdir])
def set_dir(conn, arg): git.check_repo_or_die(arg) # OK. we now know the path is a proper repository. Record this path in the # environment so that subprocesses inherit it and know where to operate. os.environ['BUP_DIR'] = arg debug1('bup server: bupdir is %r\n' % git.repodir) _set_mode() conn.ok()
def run(self): self.sock.listen(5) while True: git.check_repo_or_die() top = vfs.RefList(None) th = FTPserverThread(top, self.sock.accept()) th.daemon = True th.start()
def send_index(conn, name): git.check_repo_or_die() assert(name.find('/') < 0) assert(name.endswith('.idx')) idx = git.open_idx(git.repo('objects/pack/%s' % name)) conn.write(struct.pack('!I', len(idx.map))) conn.write(idx.map) conn.ok()
def _init_session(reinit_with_new_repopath=None): if reinit_with_new_repopath is None and git.repodir: return git.check_repo_or_die(reinit_with_new_repopath) # OK. we now know the path is a proper repository. Record this path in the # environment so that subprocesses inherit it and know where to operate. os.environ['BUP_DIR'] = git.repodir debug1('bup server: bupdir is %r\n' % git.repodir) _set_mode()
def list_indexes(conn, junk): git.check_repo_or_die() suffix = '' if dumb_server_mode: suffix = ' load' for f in os.listdir(git.repo('objects/pack')): if f.endswith('.idx'): conn.write('%s%s\n' % (f, suffix)) conn.ok()
def receive_objects_v2(conn, junk): global suspended_w git.check_repo_or_die() suggested = set() if suspended_w: w = suspended_w suspended_w = None else: if dumb_server_mode: w = git.PackWriter(objcache_maker=None) else: w = git.PackWriter() while 1: ns = conn.read(4) if not ns: w.abort() raise Exception('object read: expected length header, got EOF\n') n = struct.unpack('!I', ns)[0] #debug2('expecting %d bytes\n' % n) if not n: debug1('bup server: received %d object%s.\n' % (w.count, w.count!=1 and "s" or '')) fullpath = w.close(run_midx=not dumb_server_mode) if fullpath: (dir, name) = os.path.split(fullpath) conn.write('%s.idx\n' % name) conn.ok() return elif n == 0xffffffff: debug2('bup server: receive-objects suspended.\n') suspended_w = w conn.ok() return shar = conn.read(20) crcr = struct.unpack('!I', conn.read(4))[0] n -= 20 + 4 buf = conn.read(n) # object sizes in bup are reasonably small #debug2('read %d bytes\n' % n) _check(w, n, len(buf), 'object read: expected %d bytes, got %d\n') if not dumb_server_mode: oldpack = w.exists(shar, want_source=True) if oldpack: assert(not oldpack == True) assert(oldpack.endswith('.idx')) (dir,name) = os.path.split(oldpack) if not (name in suggested): debug1("bup server: suggesting index %s\n" % git.shorten_hash(name)) debug1("bup server: because of object %s\n" % shar.encode('hex')) conn.write('index %s\n' % name) suggested.add(name) continue nw, crc = w._raw_write((buf,), sha=shar) _check(w, crcr, crc, 'object read: expected crc %d, got %d\n')
def cat(conn, id): git.check_repo_or_die() try: for blob in git.cat(id): conn.write(struct.pack('!I', len(blob))) conn.write(blob) except KeyError, e: log('server: error: %s\n' % e) conn.write('\0\0\0\0') conn.error(e)
def test_check_repo_or_die(): git.check_repo_or_die() WVPASS('check_repo_or_die') # if we reach this point the call above passed os.rename('pybuptest.tmp/objects/pack', 'pybuptest.tmp/objects/pack.tmp') open('pybuptest.tmp/objects/pack', 'w').close() try: git.check_repo_or_die() except SystemExit, e: WVPASSEQ(e.code, 14)
def test_commit_parsing(): def showval(commit, val): return readpipe(['git', 'show', '-s', '--pretty=format:%s' % val, commit]).strip() initial_failures = wvfailure_count() orig_cwd = os.getcwd() tmpdir = tempfile.mkdtemp(dir=bup_tmp, prefix='bup-tgit-') workdir = tmpdir + "/work" repodir = workdir + '/.git' try: readpipe(['git', 'init', workdir]) os.environ['GIT_DIR'] = os.environ['BUP_DIR'] = repodir git.check_repo_or_die(repodir) os.chdir(workdir) with open('foo', 'w') as f: print >> f, 'bar' readpipe(['git', 'add', '.']) readpipe(['git', 'commit', '-am', 'Do something', '--author', 'Someone <someone@somewhere>', '--date', 'Sat Oct 3 19:48:49 2009 -0400']) commit = readpipe(['git', 'show-ref', '-s', 'master']).strip() parents = showval(commit, '%P') tree = showval(commit, '%T') cname = showval(commit, '%cn') cmail = showval(commit, '%ce') cdate = showval(commit, '%ct') coffs = showval(commit, '%ci') coffs = coffs[-5:] coff = (int(coffs[-4:-2]) * 60 * 60) + (int(coffs[-2:]) * 60) if coffs[-5] == '-': coff = - coff commit_items = git.get_commit_items(commit, git.cp()) WVPASSEQ(commit_items.parents, []) WVPASSEQ(commit_items.tree, tree) WVPASSEQ(commit_items.author_name, 'Someone') WVPASSEQ(commit_items.author_mail, 'someone@somewhere') WVPASSEQ(commit_items.author_sec, 1254613729) WVPASSEQ(commit_items.author_offset, -(4 * 60 * 60)) WVPASSEQ(commit_items.committer_name, cname) WVPASSEQ(commit_items.committer_mail, cmail) WVPASSEQ(commit_items.committer_sec, int(cdate)) WVPASSEQ(commit_items.committer_offset, coff) WVPASSEQ(commit_items.message, 'Do something\n') with open('bar', 'w') as f: print >> f, 'baz' readpipe(['git', 'add', '.']) readpipe(['git', 'commit', '-am', 'Do something else']) child = readpipe(['git', 'show-ref', '-s', 'master']).strip() parents = showval(child, '%P') commit_items = git.get_commit_items(child, git.cp()) WVPASSEQ(commit_items.parents, [commit]) finally: os.chdir(orig_cwd) if wvfailure_count() == initial_failures: subprocess.call(['rm', '-rf', tmpdir])
def cat(conn, id): global cat_pipe git.check_repo_or_die() if not cat_pipe: cat_pipe = git.CatPipe() try: for blob in cat_pipe.join(id): conn.write(struct.pack('!I', len(blob))) conn.write(blob) except KeyError, e: log('server: error: %s\n' % e) conn.write('\0\0\0\0') conn.error(e)
def test_check_repo_or_die(): initial_failures = wvfailure_count() orig_cwd = os.getcwd() tmpdir = tempfile.mkdtemp(dir=bup_tmp, prefix='bup-tgit-') os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup" try: os.chdir(tmpdir) git.init_repo(bupdir) git.check_repo_or_die() WVPASS('check_repo_or_die') # if we reach this point the call above passed os.rename(bupdir + '/objects/pack', bupdir + '/objects/pack.tmp') open(bupdir + '/objects/pack', 'w').close() try: git.check_repo_or_die() except SystemExit, e: WVPASSEQ(e.code, 14) else:
def test_check_repo_or_die(): initial_failures = wvfailure_count() orig_cwd = os.getcwd() tmpdir = tempfile.mkdtemp(dir=bup_tmp, prefix='bup-tgit-') os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup" try: os.chdir(tmpdir) git.init_repo(bupdir) git.check_repo_or_die() WVPASS('check_repo_or_die') # if we reach this point the call above passed os.rename(bupdir + '/objects/pack', bupdir + '/objects/pack.tmp') open(bupdir + '/objects/pack', 'w').close() try: git.check_repo_or_die() except SystemExit as e: WVPASSEQ(e.code, 14) else: WVFAIL() os.unlink(bupdir + '/objects/pack') os.rename(bupdir + '/objects/pack.tmp', bupdir + '/objects/pack') try: git.check_repo_or_die('nonexistantbup.tmp') except SystemExit as e: WVPASSEQ(e.code, 15) else: WVFAIL() finally: os.chdir(orig_cwd) if wvfailure_count() == initial_failures: subprocess.call(['rm', '-rf', tmpdir])
def test_check_repo_or_die(): with no_lingering_errors(), test_tempdir('bup-tgit-') as tmpdir: os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup" orig_cwd = os.getcwd() try: os.chdir(tmpdir) git.init_repo(bupdir) git.check_repo_or_die() WVPASS('check_repo_or_die') # if we reach this point the call above passed os.rename(bupdir + '/objects/pack', bupdir + '/objects/pack.tmp') open(bupdir + '/objects/pack', 'w').close() try: git.check_repo_or_die() except SystemExit as e: WVPASSEQ(e.code, 14) else: WVFAIL() os.unlink(bupdir + '/objects/pack') os.rename(bupdir + '/objects/pack.tmp', bupdir + '/objects/pack') try: git.check_repo_or_die('nonexistantbup.tmp') except SystemExit as e: WVPASSEQ(e.code, 15) else: WVFAIL() finally: os.chdir(orig_cwd)
def test_metadata_method(): with no_lingering_errors(): with test_tempdir('bup-tmetadata-') as tmpdir: bup_dir = tmpdir + '/bup' data_path = tmpdir + '/foo' os.mkdir(data_path) ex('touch', data_path + '/file') ex('ln', '-s', 'file', data_path + '/symlink') test_time1 = 13 * 1000000000 test_time2 = 42 * 1000000000 utime(data_path + '/file', (0, test_time1)) lutime(data_path + '/symlink', (0, 0)) utime(data_path, (0, test_time2)) ex(bup_path, '-d', bup_dir, 'init') ex(bup_path, '-d', bup_dir, 'index', '-v', data_path) ex(bup_path, '-d', bup_dir, 'save', '-tvvn', 'test', data_path) git.check_repo_or_die(bup_dir) repo = LocalRepo() resolved = vfs.resolve(repo, '/test/latest' + resolve_parent(data_path), follow=False) leaf_name, leaf_item = resolved[-1] m = leaf_item.meta WVPASS(m.mtime == test_time2) WVPASS(leaf_name == 'foo') contents = tuple(vfs.contents(repo, leaf_item)) WVPASS(len(contents) == 3) WVPASSEQ(frozenset(name for name, item in contents), frozenset(('.', 'file', 'symlink'))) for name, item in contents: if name == 'file': m = item.meta WVPASS(m.mtime == test_time1) elif name == 'symlink': m = item.meta WVPASSEQ(m.symlink_target, 'file') WVPASSEQ(m.size, 4) WVPASSEQ(m.mtime, 0)
def update_ref(conn, refname): git.check_repo_or_die() newval = conn.readline().strip() oldval = conn.readline().strip() git.update_ref(refname, newval.decode('hex'), oldval.decode('hex')) conn.ok()
def main(argv): handle_ctrl_c() opt = opts_from_cmdline(argv) client.bwlimit = opt.bwlimit git.check_repo_or_die() remote_dest = opt.remote or opt.is_reverse if not remote_dest: repo = git cli = nullcontext() else: try: cli = repo = client.Client(opt.remote) except client.ClientError as e: log('error: %s' % e) sys.exit(1) # cli creation must be last nontrivial command in each if clause above with cli: if not remote_dest: w = git.PackWriter(compression_level=opt.compress) else: w = cli.new_packwriter(compression_level=opt.compress) with w: sys.stdout.flush() out = byte_stream(sys.stdout) if opt.name: refname = b'refs/heads/%s' % opt.name parent = repo.read_ref(refname) else: refname = parent = None indexfile = opt.indexfile or git.repo(b'bupindex') try: msr = index.MetaStoreReader(indexfile + b'.meta') except IOError as ex: if ex.errno != ENOENT: raise log('error: cannot access %r; have you run bup index?' % path_msg(indexfile)) sys.exit(1) with msr, \ hlinkdb.HLinkDB(indexfile + b'.hlink') as hlink_db, \ index.Reader(indexfile) as reader: tree = save_tree(opt, reader, hlink_db, msr, w) if opt.tree: out.write(hexlify(tree)) out.write(b'\n') if opt.commit or opt.name: commit = commit_tree(tree, parent, opt.date, argv, w) if opt.commit: out.write(hexlify(commit)) out.write(b'\n') # packwriter must be closed before we can update the ref if opt.name: repo.update_ref(refname, commit, parent) if saved_errors: log('WARNING: %d errors encountered while saving.\n' % len(saved_errors)) sys.exit(1)
def main(argv): o = options.Options(optspec) opt, flags, extra = o.parse_bytes(argv[1:]) git.check_repo_or_die() tags = [t for sublist in git.tags().values() for t in sublist] if opt.delete: # git.delete_ref() doesn't complain if a ref doesn't exist. We # could implement this verification but we'd need to read in the # contents of the tag file and pass the hash, and we already know # about the tag's existance via "tags". tag_name = argv_bytes(opt.delete) if not opt.force and tag_name not in tags: log("error: tag '%s' doesn't exist\n" % path_msg(tag_name)) sys.exit(1) tag_file = b'refs/tags/%s' % tag_name git.delete_ref(tag_file) sys.exit(0) if not extra: for t in tags: sys.stdout.flush() out = byte_stream(sys.stdout) out.write(t) out.write(b'\n') sys.exit(0) elif len(extra) != 2: o.fatal('expected commit ref and hash') tag_name, commit = map(argv_bytes, extra[:2]) if not tag_name: o.fatal("tag name must not be empty.") debug1("args: tag name = %s; commit = %s\n" % (path_msg(tag_name), commit.decode('ascii'))) if tag_name in tags and not opt.force: log("bup: error: tag '%s' already exists\n" % path_msg(tag_name)) sys.exit(1) if tag_name.startswith(b'.'): o.fatal("'%s' is not a valid tag name." % path_msg(tag_name)) try: hash = git.rev_parse(commit) except git.GitError as e: log("bup: error: %s" % e) sys.exit(2) if not hash: log("bup: error: commit %s not found.\n" % commit.decode('ascii')) sys.exit(2) pL = git.PackIdxList(git.repo(b'objects/pack')) if not pL.exists(hash): log("bup: error: commit %s not found.\n" % commit.decode('ascii')) sys.exit(2) tag_file = git.repo(b'refs/tags/' + tag_name) try: tag = open(tag_file, 'wb') except OSError as e: log("bup: error: could not create tag '%s': %s" % (path_msg(tag_name), e)) sys.exit(3) with tag as tag: tag.write(hexlify(hash)) tag.write(b'\n')
git.check_repo_or_die() WVPASS('check_repo_or_die') # if we reach this point the call above passed os.rename(bupdir + '/objects/pack', bupdir + '/objects/pack.tmp') open(bupdir + '/objects/pack', 'w').close() try: git.check_repo_or_die() except SystemExit, e: WVPASSEQ(e.code, 14) else: WVFAIL() os.unlink(bupdir + '/objects/pack') os.rename(bupdir + '/objects/pack.tmp', bupdir + '/objects/pack') try: git.check_repo_or_die('nonexistantbup.tmp') except SystemExit, e: WVPASSEQ(e.code, 15) else: WVFAIL() finally: os.chdir(orig_cwd) if wvfailure_count() == initial_failures: subprocess.call(['rm', '-rf', tmpdir]) @wvtest def test_commit_parsing(): def showval(commit, val): return readpipe(['git', 'show', '-s', '--pretty=format:%s' % val, commit]).strip()
from bup.git import check_repo_or_die from bup.options import Options from bup.helpers import die_if_errors, handle_ctrl_c, log from bup.repo import LocalRepo from bup.rm import bup_rm optspec = """ bup rm <branch|save...> -- #,compress= set compression level to # (0-9, 9 is highest) [6] v,verbose increase verbosity (can be specified multiple times) unsafe use the command even though it may be DANGEROUS """ handle_ctrl_c() o = Options(optspec) opt, flags, extra = o.parse(sys.argv[1:]) if not opt.unsafe: o.fatal('refusing to run dangerous, experimental command without --unsafe') if len(extra) < 1: o.fatal('no paths specified') check_repo_or_die() repo = LocalRepo() bup_rm(repo, extra, compression=opt.compress, verbosity=opt.verbose) die_if_errors()
def set_dir(conn, arg): git.check_repo_or_die(arg) debug1('bup server: bupdir is %r\n' % git.repodir) _set_mode() conn.ok()
def main(): o = options.Options(optspec) opt, flags, extra = o.parse(sys.argv[1:]) verbosity = (opt.verbose or 0) if not opt.quiet else -1 if opt.remote: opt.remote = argv_bytes(opt.remote) if opt.outdir: opt.outdir = argv_bytes(opt.outdir) git.check_repo_or_die() if not extra: o.fatal('must specify at least one filename to restore') exclude_rxs = parse_rx_excludes(flags, o.fatal) owner_map = {} for map_type in ('user', 'group', 'uid', 'gid'): owner_map[map_type] = parse_owner_mappings(map_type, flags, o.fatal) if opt.outdir: mkdirp(opt.outdir) os.chdir(opt.outdir) if opt.remote: opt.remote = argv_bytes(opt.remote) repo = from_opts(opt, reverse=False) top = fsencode(os.getcwd()) hardlinks = {} for path in [argv_bytes(x) for x in extra]: if not valid_restore_path(path): add_error("path %r doesn't include a branch and revision" % path) continue try: resolved = vfs.resolve(repo, path, want_meta=True, follow=False) except vfs.IOError as e: add_error(e) continue if len(resolved) == 3 and resolved[2][0] == b'latest': # Follow latest symlink to the actual save try: resolved = vfs.resolve(repo, b'latest', parent=resolved[:-1], want_meta=True) except vfs.IOError as e: add_error(e) continue # Rename it back to 'latest' resolved = tuple(elt if i != 2 else (b'latest', ) + elt[1:] for i, elt in enumerate(resolved)) path_parent, path_name = os.path.split(path) leaf_name, leaf_item = resolved[-1] if not leaf_item: add_error('error: cannot access %r in %r' % ('/'.join(name for name, item in resolved), path)) continue if not path_name or path_name == b'.': # Source is /foo/what/ever/ or /foo/what/ever/. -- extract # what/ever/* to the current directory, and if name == '.' # (i.e. /foo/what/ever/.), then also restore what/ever's # metadata to the current directory. treeish = vfs.item_mode(leaf_item) if not treeish: add_error('%r cannot be restored as a directory' % path) else: items = vfs.contents(repo, leaf_item, want_meta=True) dot, leaf_item = next(items, None) assert dot == b'.' for sub_name, sub_item in items: restore(repo, b'', sub_name, sub_item, top, opt.sparse, opt.numeric_ids, owner_map, exclude_rxs, verbosity, hardlinks) if path_name == b'.': leaf_item = vfs.augment_item_meta(repo, leaf_item, include_size=True) apply_metadata(leaf_item.meta, b'.', opt.numeric_ids, owner_map) else: restore(repo, b'', leaf_name, leaf_item, top, opt.sparse, opt.numeric_ids, owner_map, exclude_rxs, verbosity, hardlinks) if verbosity >= 0: progress('Restoring: %d, done.\n' % total_restored) die_if_errors()
def main(): o = options.Options(optspec) opt, flags, extra = o.parse(sys.argv[1:]) verbosity = opt.verbose if not opt.quiet else -1 git.check_repo_or_die() if not extra: o.fatal('must specify at least one filename to restore') exclude_rxs = parse_rx_excludes(flags, o.fatal) owner_map = {} for map_type in ('user', 'group', 'uid', 'gid'): owner_map[map_type] = parse_owner_mappings(map_type, flags, o.fatal) if opt.outdir: mkdirp(opt.outdir) os.chdir(opt.outdir) repo = RemoteRepo(opt.remote) if opt.remote else LocalRepo() top = os.getcwd() hardlinks = {} for path in extra: if not valid_restore_path(path): add_error("path %r doesn't include a branch and revision" % path) continue try: resolved = vfs.lresolve(repo, path, want_meta=True) except vfs.IOError as e: add_error(e) continue path_parent, path_name = os.path.split(path) leaf_name, leaf_item = resolved[-1] if not leaf_item: add_error('error: cannot access %r in %r' % ('/'.join(name for name, item in resolved), path)) continue if not path_name or path_name == '.': # Source is /foo/what/ever/ or /foo/what/ever/. -- extract # what/ever/* to the current directory, and if name == '.' # (i.e. /foo/what/ever/.), then also restore what/ever's # metadata to the current directory. treeish = vfs.item_mode(leaf_item) if not treeish: add_error('%r cannot be restored as a directory' % path) else: items = vfs.contents(repo, leaf_item, want_meta=True) dot, leaf_item = next(items, None) assert (dot == '.') for sub_name, sub_item in items: restore(repo, '', sub_name, sub_item, top, opt.sparse, opt.numeric_ids, owner_map, exclude_rxs, verbosity, hardlinks) if path_name == '.': leaf_item = vfs.augment_item_meta(repo, leaf_item, include_size=True) apply_metadata(leaf_item.meta, '.', opt.numeric_ids, owner_map) else: restore(repo, '', leaf_name, leaf_item, top, opt.sparse, opt.numeric_ids, owner_map, exclude_rxs, verbosity, hardlinks) if verbosity >= 0: progress('Restoring: %d, done.\n' % total_restored) die_if_errors()
def main(argv): global opt, par2_ok o = options.Options(optspec) opt, flags, extra = o.parse_bytes(argv[1:]) opt.verbose = opt.verbose or 0 par2_setup() if opt.par2_ok: if par2_ok: sys.exit(0) # 'true' in sh else: sys.exit(1) if opt.disable_par2: par2_ok = 0 git.check_repo_or_die() if extra: extra = [argv_bytes(x) for x in extra] else: debug('fsck: No filenames given: checking all packs.\n') extra = glob.glob(git.repo(b'objects/pack/*.pack')) sys.stdout.flush() out = byte_stream(sys.stdout) code = 0 count = 0 outstanding = {} for name in extra: if name.endswith(b'.pack'): base = name[:-5] elif name.endswith(b'.idx'): base = name[:-4] elif name.endswith(b'.par2'): base = name[:-5] elif os.path.exists(name + b'.pack'): base = name else: raise Exception('%r is not a pack file!' % name) (dir, last) = os.path.split(base) par2_exists = os.path.exists(base + b'.par2') if par2_exists and os.stat(base + b'.par2').st_size == 0: par2_exists = 0 sys.stdout.flush( ) # Not sure we still need this, but it'll flush out too debug('fsck: checking %r (%s)\n' % (last, par2_ok and par2_exists and 'par2' or 'git')) if not opt.verbose: progress('fsck (%d/%d)\r' % (count, len(extra))) if not opt.jobs: nc = do_pack(base, last, par2_exists, out) code = code or nc count += 1 else: while len(outstanding) >= opt.jobs: (pid, nc) = os.wait() nc >>= 8 if pid in outstanding: del outstanding[pid] code = code or nc count += 1 pid = os.fork() if pid: # parent outstanding[pid] = 1 else: # child try: sys.exit(do_pack(base, last, par2_exists, out)) except Exception as e: log('exception: %r\n' % e) sys.exit(99) while len(outstanding): (pid, nc) = os.wait() nc >>= 8 if pid in outstanding: del outstanding[pid] code = code or nc count += 1 if not opt.verbose: progress('fsck (%d/%d)\r' % (count, len(extra))) if istty2: debug('fsck done. \n') sys.exit(code)
def list_indexes(conn, junk): git.check_repo_or_die() for f in os.listdir(git.repo('objects/pack')): if f.endswith('.idx'): conn.write('%s\n' % f) conn.ok()
def set_dir(conn, arg): git.check_repo_or_die(arg) log('bup server: bupdir is %r\n' % git.repodir) conn.ok()
def test_commit_parsing(tmpdir): def restore_env_var(name, val): if val is None: del environ[name] else: environ[name] = val def showval(commit, val): return readpipe( [b'git', b'show', b'-s', b'--pretty=format:%s' % val, commit]).strip() orig_cwd = os.getcwd() workdir = tmpdir + b'/work' repodir = workdir + b'/.git' orig_author_name = environ.get(b'GIT_AUTHOR_NAME') orig_author_email = environ.get(b'GIT_AUTHOR_EMAIL') orig_committer_name = environ.get(b'GIT_COMMITTER_NAME') orig_committer_email = environ.get(b'GIT_COMMITTER_EMAIL') environ[b'GIT_AUTHOR_NAME'] = b'bup test' environ[b'GIT_COMMITTER_NAME'] = environ[b'GIT_AUTHOR_NAME'] environ[b'GIT_AUTHOR_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f' environ[b'GIT_COMMITTER_EMAIL'] = environ[b'GIT_AUTHOR_EMAIL'] try: readpipe([b'git', b'init', workdir]) environ[b'GIT_DIR'] = environ[b'BUP_DIR'] = repodir git.check_repo_or_die(repodir) os.chdir(workdir) with open('foo', 'w') as f: print('bar', file=f) readpipe([b'git', b'add', b'.']) readpipe([ b'git', b'commit', b'-am', b'Do something', b'--author', b'Someone <someone@somewhere>', b'--date', b'Sat Oct 3 19:48:49 2009 -0400' ]) commit = readpipe([b'git', b'show-ref', b'-s', b'master']).strip() parents = showval(commit, b'%P') tree = showval(commit, b'%T') cname = showval(commit, b'%cn') cmail = showval(commit, b'%ce') cdate = showval(commit, b'%ct') coffs = showval(commit, b'%ci') coffs = coffs[-5:] coff = (int(coffs[-4:-2]) * 60 * 60) + (int(coffs[-2:]) * 60) if bytes_from_byte(coffs[-5]) == b'-': coff = -coff commit_items = git.get_commit_items(commit, git.cp()) WVPASSEQ(commit_items.parents, []) WVPASSEQ(commit_items.tree, tree) WVPASSEQ(commit_items.author_name, b'Someone') WVPASSEQ(commit_items.author_mail, b'someone@somewhere') WVPASSEQ(commit_items.author_sec, 1254613729) WVPASSEQ(commit_items.author_offset, -(4 * 60 * 60)) WVPASSEQ(commit_items.committer_name, cname) WVPASSEQ(commit_items.committer_mail, cmail) WVPASSEQ(commit_items.committer_sec, int(cdate)) WVPASSEQ(commit_items.committer_offset, coff) WVPASSEQ(commit_items.message, b'Do something\n') with open(b'bar', 'wb') as f: f.write(b'baz\n') readpipe([b'git', b'add', '.']) readpipe([b'git', b'commit', b'-am', b'Do something else']) child = readpipe([b'git', b'show-ref', b'-s', b'master']).strip() parents = showval(child, b'%P') commit_items = git.get_commit_items(child, git.cp()) WVPASSEQ(commit_items.parents, [commit]) finally: os.chdir(orig_cwd) restore_env_var(b'GIT_AUTHOR_NAME', orig_author_name) restore_env_var(b'GIT_AUTHOR_EMAIL', orig_author_email) restore_env_var(b'GIT_COMMITTER_NAME', orig_committer_name) restore_env_var(b'GIT_COMMITTER_EMAIL', orig_committer_email)
def main(argv): global opt signal.signal(signal.SIGTERM, handle_sigterm) UnixAddress = namedtuple('UnixAddress', ['path']) InetAddress = namedtuple('InetAddress', ['host', 'port']) o = options.Options(optspec) opt, flags, extra = o.parse_bytes(argv[1:]) if len(extra) > 1: o.fatal("at most one argument expected") if len(extra) == 0: address = InetAddress(host='127.0.0.1', port=8080) else: bind_url = extra[0] if bind_url.startswith('unix://'): address = UnixAddress(path=bind_url[len('unix://'):]) else: addr_parts = extra[0].split(':', 1) if len(addr_parts) == 1: host = '127.0.0.1' port = addr_parts[0] else: host, port = addr_parts try: port = int(port) except (TypeError, ValueError) as ex: o.fatal('port must be an integer, not %r' % port) address = InetAddress(host=host, port=port) git.check_repo_or_die() settings = dict( debug=1, template_path=resource_path(b'web').decode('utf-8'), static_path=resource_path(b'web/static').decode('utf-8'), ) # Disable buffering on stdout, for debug messages try: sys.stdout._line_buffering = True except AttributeError: sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) application = tornado.web.Application([ (r"(?P<path>/.*)", BupRequestHandler, dict(repo=LocalRepo())), ], **settings) http_server = HTTPServer(application) io_loop_pending = IOLoop.instance() if isinstance(address, InetAddress): sockets = tornado.netutil.bind_sockets(address.port, address.host) http_server.add_sockets(sockets) print('Serving HTTP on %s:%d...' % sockets[0].getsockname()[0:2]) if opt.browser: browser_addr = 'http://' + address[0] + ':' + str(address[1]) io_loop_pending.add_callback(lambda: webbrowser.open(browser_addr)) elif isinstance(address, UnixAddress): unix_socket = bind_unix_socket(address.path) http_server.add_socket(unix_socket) print('Serving HTTP on filesystem socket %r' % address.path) else: log('error: unexpected address %r', address) sys.exit(1) io_loop = io_loop_pending io_loop.start() if saved_errors: log('WARNING: %d errors encountered while saving.\n' % len(saved_errors)) sys.exit(1)
def read_ref(conn, refname): git.check_repo_or_die() r = git.read_ref(refname) conn.write('%s\n' % (r or '').encode('hex')) conn.ok()
def main(): handle_ctrl_c() is_reverse = environ.get(b'BUP_SERVER_REVERSE') opt = parse_args(sys.argv) git.check_repo_or_die() src_dir = opt.source or git.repo() if opt.bwlimit: client.bwlimit = parse_num(opt.bwlimit) if is_reverse and opt.remote: misuse("don't use -r in reverse mode; it's automatic") if opt.remote: opt.remote = argv_bytes(opt.remote) if opt.remote or is_reverse: dest_repo = RemoteRepo(opt.remote) else: dest_repo = LocalRepo() with dest_repo as dest_repo: with LocalRepo(repo_dir=src_dir) as src_repo: with dest_repo.new_packwriter(compression_level=opt.compress) as writer: # Resolve and validate all sources and destinations, # implicit or explicit, and do it up-front, so we can # fail before we start writing (for any obviously # broken cases). target_items = resolve_targets(opt.target_specs, src_repo, dest_repo) updated_refs = {} # ref_name -> (original_ref, tip_commit(bin)) no_ref_info = (None, None) handlers = {'ff': handle_ff, 'append': handle_append, 'force-pick': handle_pick, 'pick': handle_pick, 'new-tag': handle_new_tag, 'replace': handle_replace, 'unnamed': handle_unnamed} for item in target_items: debug1('get-spec: %r\n' % (item.spec,)) debug1('get-src: %s\n' % loc_desc(item.src)) debug1('get-dest: %s\n' % loc_desc(item.dest)) dest_path = item.dest and item.dest.path if dest_path: if dest_path.startswith(b'/.tag/'): dest_ref = b'refs/tags/%s' % dest_path[6:] else: dest_ref = b'refs/heads/%s' % dest_path[1:] else: dest_ref = None dest_hash = item.dest and item.dest.hash orig_ref, cur_ref = updated_refs.get(dest_ref, no_ref_info) orig_ref = orig_ref or dest_hash cur_ref = cur_ref or dest_hash handler = handlers[item.spec.method] item_result = handler(item, src_repo, writer, opt) if len(item_result) > 1: new_id, tree = item_result else: new_id = item_result[0] if not dest_ref: log_item(item.spec.src, item.src.type, opt) else: updated_refs[dest_ref] = (orig_ref, new_id) if dest_ref.startswith(b'refs/tags/'): log_item(item.spec.src, item.src.type, opt, tag=new_id) else: log_item(item.spec.src, item.src.type, opt, tree=tree, commit=new_id) # Only update the refs at the very end, once the writer is # closed, so that if something goes wrong above, the old refs # will be undisturbed. for ref_name, info in items(updated_refs): orig_ref, new_ref = info try: dest_repo.update_ref(ref_name, new_ref, orig_ref) if opt.verbose: new_hex = hexlify(new_ref) if orig_ref: orig_hex = hexlify(orig_ref) log('updated %r (%s -> %s)\n' % (ref_name, orig_hex, new_hex)) else: log('updated %r (%s)\n' % (ref_name, new_hex)) except (git.GitError, client.ClientError) as ex: add_error('unable to update ref %r: %s' % (ref_name, ex)) if saved_errors: log('WARNING: %d errors encountered while saving.\n' % len(saved_errors)) sys.exit(1)
def main(): o = options.Options(optspec) opt, flags, extra = o.parse(sys.argv[1:]) verbosity = opt.verbose if not opt.quiet else -1 git.check_repo_or_die() if not extra: o.fatal('must specify at least one filename to restore') exclude_rxs = parse_rx_excludes(flags, o.fatal) owner_map = {} for map_type in ('user', 'group', 'uid', 'gid'): owner_map[map_type] = parse_owner_mappings(map_type, flags, o.fatal) if opt.outdir: mkdirp(opt.outdir) os.chdir(opt.outdir) repo = RemoteRepo(opt.remote) if opt.remote else LocalRepo() top = os.getcwd() hardlinks = {} for path in extra: if not valid_restore_path(path): add_error("path %r doesn't include a branch and revision" % path) continue try: resolved = vfs.resolve(repo, path, want_meta=True, follow=False) except vfs.IOError as e: add_error(e) continue if len(resolved) == 3 and resolved[2][0] == 'latest': # Follow latest symlink to the actual save try: resolved = vfs.resolve(repo, 'latest', parent=resolved[:-1], want_meta=True) except vfs.IOError as e: add_error(e) continue # Rename it back to 'latest' resolved = tuple(elt if i != 2 else ('latest',) + elt[1:] for i, elt in enumerate(resolved)) path_parent, path_name = os.path.split(path) leaf_name, leaf_item = resolved[-1] if not leaf_item: add_error('error: cannot access %r in %r' % ('/'.join(name for name, item in resolved), path)) continue if not path_name or path_name == '.': # Source is /foo/what/ever/ or /foo/what/ever/. -- extract # what/ever/* to the current directory, and if name == '.' # (i.e. /foo/what/ever/.), then also restore what/ever's # metadata to the current directory. treeish = vfs.item_mode(leaf_item) if not treeish: add_error('%r cannot be restored as a directory' % path) else: items = vfs.contents(repo, leaf_item, want_meta=True) dot, leaf_item = next(items, None) assert(dot == '.') for sub_name, sub_item in items: restore(repo, '', sub_name, sub_item, top, opt.sparse, opt.numeric_ids, owner_map, exclude_rxs, verbosity, hardlinks) if path_name == '.': leaf_item = vfs.augment_item_meta(repo, leaf_item, include_size=True) apply_metadata(leaf_item.meta, '.', opt.numeric_ids, owner_map) else: restore(repo, '', leaf_name, leaf_item, top, opt.sparse, opt.numeric_ids, owner_map, exclude_rxs, verbosity, hardlinks) if verbosity >= 0: progress('Restoring: %d, done.\n' % total_restored) die_if_errors()
def create(self, repo_dir=None): # FIXME: this is not ideal, we should somehow # be able to call the constructor instead? git.init_repo(repo_dir) git.check_repo_or_die(repo_dir)
WVPASS('check_repo_or_die' ) # if we reach this point the call above passed os.rename(bupdir + '/objects/pack', bupdir + '/objects/pack.tmp') open(bupdir + '/objects/pack', 'w').close() try: git.check_repo_or_die() except SystemExit, e: WVPASSEQ(e.code, 14) else: WVFAIL() os.unlink(bupdir + '/objects/pack') os.rename(bupdir + '/objects/pack.tmp', bupdir + '/objects/pack') try: git.check_repo_or_die('nonexistantbup.tmp') except SystemExit, e: WVPASSEQ(e.code, 15) else: WVFAIL() finally: os.chdir(orig_cwd) if wvfailure_count() == initial_failures: subprocess.call(['rm', '-rf', tmpdir]) @wvtest def test_commit_parsing(): def restore_env_var(name, val): if val is None: del os.environ[name]
def test_commit_parsing(): def restore_env_var(name, val): if val is None: del os.environ[name] else: os.environ[name] = val def showval(commit, val): return readpipe( ['git', 'show', '-s', '--pretty=format:%s' % val, commit]).strip() initial_failures = wvfailure_count() orig_cwd = os.getcwd() tmpdir = tempfile.mkdtemp(dir=bup_tmp, prefix='bup-tgit-') workdir = tmpdir + "/work" repodir = workdir + '/.git' orig_author_name = os.environ.get('GIT_AUTHOR_NAME') orig_author_email = os.environ.get('GIT_AUTHOR_EMAIL') orig_committer_name = os.environ.get('GIT_COMMITTER_NAME') orig_committer_email = os.environ.get('GIT_COMMITTER_EMAIL') os.environ['GIT_AUTHOR_NAME'] = 'bup test' os.environ['GIT_COMMITTER_NAME'] = os.environ['GIT_AUTHOR_NAME'] os.environ['GIT_AUTHOR_EMAIL'] = 'bup@a425bc70a02811e49bdf73ee56450e6f' os.environ['GIT_COMMITTER_EMAIL'] = os.environ['GIT_AUTHOR_EMAIL'] try: readpipe(['git', 'init', workdir]) os.environ['GIT_DIR'] = os.environ['BUP_DIR'] = repodir git.check_repo_or_die(repodir) os.chdir(workdir) with open('foo', 'w') as f: print >> f, 'bar' readpipe(['git', 'add', '.']) readpipe([ 'git', 'commit', '-am', 'Do something', '--author', 'Someone <someone@somewhere>', '--date', 'Sat Oct 3 19:48:49 2009 -0400' ]) commit = readpipe(['git', 'show-ref', '-s', 'master']).strip() parents = showval(commit, '%P') tree = showval(commit, '%T') cname = showval(commit, '%cn') cmail = showval(commit, '%ce') cdate = showval(commit, '%ct') coffs = showval(commit, '%ci') coffs = coffs[-5:] coff = (int(coffs[-4:-2]) * 60 * 60) + (int(coffs[-2:]) * 60) if coffs[-5] == '-': coff = -coff commit_items = git.get_commit_items(commit, git.cp()) WVPASSEQ(commit_items.parents, []) WVPASSEQ(commit_items.tree, tree) WVPASSEQ(commit_items.author_name, 'Someone') WVPASSEQ(commit_items.author_mail, 'someone@somewhere') WVPASSEQ(commit_items.author_sec, 1254613729) WVPASSEQ(commit_items.author_offset, -(4 * 60 * 60)) WVPASSEQ(commit_items.committer_name, cname) WVPASSEQ(commit_items.committer_mail, cmail) WVPASSEQ(commit_items.committer_sec, int(cdate)) WVPASSEQ(commit_items.committer_offset, coff) WVPASSEQ(commit_items.message, 'Do something\n') with open('bar', 'w') as f: print >> f, 'baz' readpipe(['git', 'add', '.']) readpipe(['git', 'commit', '-am', 'Do something else']) child = readpipe(['git', 'show-ref', '-s', 'master']).strip() parents = showval(child, '%P') commit_items = git.get_commit_items(child, git.cp()) WVPASSEQ(commit_items.parents, [commit]) finally: os.chdir(orig_cwd) restore_env_var('GIT_AUTHOR_NAME', orig_author_name) restore_env_var('GIT_AUTHOR_EMAIL', orig_author_email) restore_env_var('GIT_COMMITTER_NAME', orig_committer_name) restore_env_var('GIT_COMMITTER_EMAIL', orig_committer_email) if wvfailure_count() == initial_failures: subprocess.call(['rm', '-rf', tmpdir])
def __init__(self, repo_dir): if opt.force_repo: repo_dir = None git.check_repo_or_die(repo_dir) LocalRepo.__init__(self, repo_dir)
def main(argv): o = options.Options(optspec) opt, flags, extra = o.parse_bytes(argv[1:]) if not (opt.modified or \ opt['print'] or \ opt.status or \ opt.update or \ opt.check or \ opt.clear): opt.update = 1 if (opt.fake_valid or opt.fake_invalid) and not opt.update: o.fatal('--fake-{in,}valid are meaningless without -u') if opt.fake_valid and opt.fake_invalid: o.fatal('--fake-valid is incompatible with --fake-invalid') if opt.clear and opt.indexfile: o.fatal('cannot clear an external index (via -f)') # FIXME: remove this once we account for timestamp races, i.e. index; # touch new-file; index. It's possible for this to happen quickly # enough that new-file ends up with the same timestamp as the first # index, and then bup will ignore it. tick_start = time.time() time.sleep(1 - (tick_start - int(tick_start))) git.check_repo_or_die() handle_ctrl_c() if opt.verbose is None: opt.verbose = 0 if opt.indexfile: indexfile = argv_bytes(opt.indexfile) else: indexfile = git.repo(b'bupindex') if opt.check: log('check: starting initial check.\n') check_index(index.Reader(indexfile), opt.verbose) if opt.clear: log('clear: clearing index.\n') clear_index(indexfile, opt.verbose) sys.stdout.flush() out = byte_stream(sys.stdout) if opt.update: if not extra: o.fatal('update mode (-u) requested but no paths given') extra = [argv_bytes(x) for x in extra] excluded_paths = parse_excludes(flags, o.fatal) exclude_rxs = parse_rx_excludes(flags, o.fatal) xexcept = index.unique_resolved_paths(extra) for rp, path in index.reduce_paths(extra): update_index(rp, excluded_paths, exclude_rxs, indexfile, check=opt.check, check_device=opt.check_device, xdev=opt.xdev, xdev_exceptions=xexcept, fake_valid=opt.fake_valid, fake_invalid=opt.fake_invalid, out=out, verbose=opt.verbose) if opt['print'] or opt.status or opt.modified: extra = [argv_bytes(x) for x in extra] for name, ent in index.Reader(indexfile).filter(extra or [b'']): if (opt.modified and (ent.is_valid() or ent.is_deleted() or not ent.mode)): continue line = b'' if opt.status: if ent.is_deleted(): line += b'D ' elif not ent.is_valid(): if ent.sha == index.EMPTY_SHA: line += b'A ' else: line += b'M ' else: line += b' ' if opt.hash: line += hexlify(ent.sha) + b' ' if opt.long: line += b'%7s %7s ' % (oct(ent.mode).encode('ascii'), oct(ent.gitmode).encode('ascii')) out.write(line + (name or b'./') + b'\n') if opt.check and (opt['print'] or opt.status or opt.modified or opt.update): log('check: starting final check.\n') check_index(index.Reader(indexfile), opt.verbose) if saved_errors: log('WARNING: %d errors encountered.\n' % len(saved_errors)) sys.exit(1)
def main(argv): o = options.Options(optspec) opt, flags, extra = o.parse_bytes(argv[1:]) git.check_repo_or_die() global repo sys.stdout.flush() out = byte_stream(sys.stdout) stdin = byte_stream(sys.stdin) repo = LocalRepo() pwd = vfs.resolve(repo, b'/') rv = 0 def inputiter(f): if os.isatty(f.fileno()): while 1: prompt = b'bup %s> ' % (b'/'.join(name for name, item in pwd) or b'/', ) if hasattr(_helpers, 'readline'): try: yield _helpers.readline(prompt) except EOFError: print() # Clear the line for the terminal's next prompt break else: out.write(prompt) out.flush() read_line = f.readline() if not read_line: print('') break yield read_line else: for line in f: yield line if extra: lines = (argv_bytes(arg) for arg in extra) else: if hasattr(_helpers, 'readline'): _helpers.set_completer_word_break_characters(b' \t\n\r/') _helpers.set_attempted_completion_function(attempt_completion) _helpers.set_completion_entry_function(enter_completion) if sys.platform.startswith('darwin'): # MacOS uses a slightly incompatible clone of libreadline _helpers.parse_and_bind(b'bind ^I rl_complete') _helpers.parse_and_bind(b'tab: complete') lines = inputiter(stdin) for line in lines: if not line.strip(): continue words = [word for (wordstart,word) in shquote.quotesplit(line)] cmd = words[0].lower() #log('execute: %r %r\n' % (cmd, parm)) try: if cmd == b'ls': do_ls(repo, pwd, words[1:], out) out.flush() elif cmd == b'cd': np = pwd for parm in words[1:]: res = vfs.resolve(repo, parm, parent=np) _, leaf_item = res[-1] if not leaf_item: raise Exception('%s does not exist' % path_msg(b'/'.join(name for name, item in res))) if not stat.S_ISDIR(vfs.item_mode(leaf_item)): raise Exception('%s is not a directory' % path_msg(parm)) np = res pwd = np elif cmd == b'pwd': if len(pwd) == 1: out.write(b'/') out.write(b'/'.join(name for name, item in pwd) + b'\n') out.flush() elif cmd == b'cat': for parm in words[1:]: res = vfs.resolve(repo, parm, parent=pwd) _, leaf_item = res[-1] if not leaf_item: raise Exception('%s does not exist' % path_msg(b'/'.join(name for name, item in res))) with vfs.fopen(repo, leaf_item) as srcfile: write_to_file(srcfile, out) out.flush() elif cmd == b'get': if len(words) not in [2,3]: rv = 1 raise Exception('Usage: get <filename> [localname]') rname = words[1] (dir,base) = os.path.split(rname) lname = len(words) > 2 and words[2] or base res = vfs.resolve(repo, rname, parent=pwd) _, leaf_item = res[-1] if not leaf_item: raise Exception('%s does not exist' % path_msg(b'/'.join(name for name, item in res))) with vfs.fopen(repo, leaf_item) as srcfile: with open(lname, 'wb') as destfile: log('Saving %s\n' % path_msg(lname)) write_to_file(srcfile, destfile) elif cmd == b'mget': for parm in words[1:]: dir, base = os.path.split(parm) res = vfs.resolve(repo, dir, parent=pwd) _, dir_item = res[-1] if not dir_item: raise Exception('%s does not exist' % path_msg(dir)) for name, item in vfs.contents(repo, dir_item): if name == b'.': continue if fnmatch.fnmatch(name, base): if stat.S_ISLNK(vfs.item_mode(item)): deref = vfs.resolve(repo, name, parent=res) deref_name, deref_item = deref[-1] if not deref_item: raise Exception('%s does not exist' % path_msg('/'.join(name for name, item in deref))) item = deref_item with vfs.fopen(repo, item) as srcfile: with open(name, 'wb') as destfile: log('Saving %s\n' % path_msg(name)) write_to_file(srcfile, destfile) elif cmd == b'help' or cmd == b'?': out.write(b'Commands: ls cd pwd cat get mget help quit\n') out.flush() elif cmd in (b'quit', b'exit', b'bye'): break else: rv = 1 raise Exception('no such command %r' % cmd) except Exception as e: rv = 1 log('error: %s\n' % e) raise sys.exit(rv)
def main(argv): # Hack around lack of nonlocal vars in python 2 _nonlocal = {} o = options.Options(optspec) opt, flags, extra = o.parse_bytes(argv[1:]) if opt.indexfile: opt.indexfile = argv_bytes(opt.indexfile) if opt.name: opt.name = argv_bytes(opt.name) if opt.remote: opt.remote = argv_bytes(opt.remote) if opt.strip_path: opt.strip_path = argv_bytes(opt.strip_path) git.check_repo_or_die() if not (opt.tree or opt.commit or opt.name): o.fatal("use one or more of -t, -c, -n") if not extra: o.fatal("no filenames given") extra = [argv_bytes(x) for x in extra] opt.progress = (istty2 and not opt.quiet) opt.smaller = parse_num(opt.smaller or 0) if opt.bwlimit: client.bwlimit = parse_num(opt.bwlimit) if opt.date: date = parse_date_or_fatal(opt.date, o.fatal) else: date = time.time() if opt.strip and opt.strip_path: o.fatal("--strip is incompatible with --strip-path") graft_points = [] if opt.graft: if opt.strip: o.fatal("--strip is incompatible with --graft") if opt.strip_path: o.fatal("--strip-path is incompatible with --graft") for (option, parameter) in flags: if option == "--graft": parameter = argv_bytes(parameter) splitted_parameter = parameter.split(b'=') if len(splitted_parameter) != 2: o.fatal("a graft point must be of the form old_path=new_path") old_path, new_path = splitted_parameter if not (old_path and new_path): o.fatal("a graft point cannot be empty") graft_points.append((resolve_parent(old_path), resolve_parent(new_path))) is_reverse = environ.get(b'BUP_SERVER_REVERSE') if is_reverse and opt.remote: o.fatal("don't use -r in reverse mode; it's automatic") name = opt.name if name and not valid_save_name(name): o.fatal("'%s' is not a valid branch name" % path_msg(name)) refname = name and b'refs/heads/%s' % name or None if opt.remote or is_reverse: try: cli = client.Client(opt.remote) except client.ClientError as e: log('error: %s' % e) sys.exit(1) oldref = refname and cli.read_ref(refname) or None w = cli.new_packwriter(compression_level=opt.compress) else: cli = None oldref = refname and git.read_ref(refname) or None w = git.PackWriter(compression_level=opt.compress) handle_ctrl_c() # Metadata is stored in a file named .bupm in each directory. The # first metadata entry will be the metadata for the current directory. # The remaining entries will be for each of the other directory # elements, in the order they're listed in the index. # # Since the git tree elements are sorted according to # git.shalist_item_sort_key, the metalist items are accumulated as # (sort_key, metadata) tuples, and then sorted when the .bupm file is # created. The sort_key should have been computed using the element's # mangled name and git mode (after hashsplitting), but the code isn't # actually doing that but rather uses the element's real name and mode. # This makes things a bit more difficult when reading it back, see # vfs.ordered_tree_entries(). # Maintain a stack of information representing the current location in # the archive being constructed. The current path is recorded in # parts, which will be something like # [StackDir(name=''), StackDir(name='home'), StackDir(name='someuser')], # and the accumulated content and metadata for files in the dirs is stored # in the .items member of the StackDir. stack = [] def _push(part, metadata): # Enter a new archive directory -- make it the current directory. item = StackDir(part, metadata) stack.append(item) def _pop(force_tree=None, dir_metadata=None): # Leave the current archive directory and add its tree to its parent. item = stack.pop() # FIXME: only test if collision is possible (i.e. given --strip, etc.)? if force_tree: tree = force_tree else: names_seen = set() clean_list = [] for x in item.items: name = x.name if name in names_seen: parent_path = b'/'.join(x.name for x in stack) + b'/' add_error('error: ignoring duplicate path %s in %s' % (path_msg(name), path_msg(parent_path))) else: names_seen.add(name) clean_list.append(x) # if set, overrides the original metadata pushed for this dir. if dir_metadata is None: dir_metadata = item.meta metalist = [(b'', dir_metadata)] metalist += [(git.shalist_item_sort_key((entry.mode, entry.name, None)), entry.meta) for entry in clean_list if entry.mode != GIT_MODE_TREE] metalist.sort(key = lambda x: x[0]) metadata = BytesIO(b''.join(m[1].encode() for m in metalist)) mode, id = hashsplit.split_to_blob_or_tree(w.new_blob, w.new_tree, [metadata], keep_boundaries=False) shalist = [(mode, b'.bupm', id)] shalist += [(entry.gitmode, git.mangle_name(entry.name, entry.mode, entry.gitmode), entry.oid) for entry in clean_list] tree = w.new_tree(shalist) if stack: stack[-1].append(item.name, GIT_MODE_TREE, GIT_MODE_TREE, tree, None) return tree _nonlocal['count'] = 0 _nonlocal['subcount'] = 0 _nonlocal['lastremain'] = None def progress_report(n): _nonlocal['subcount'] += n cc = _nonlocal['count'] + _nonlocal['subcount'] pct = total and (cc*100.0/total) or 0 now = time.time() elapsed = now - tstart kps = elapsed and int(cc/1024./elapsed) kps_frac = 10 ** int(math.log(kps+1, 10) - 1) kps = int(kps/kps_frac)*kps_frac if cc: remain = elapsed*1.0/cc * (total-cc) else: remain = 0.0 if (_nonlocal['lastremain'] and (remain > _nonlocal['lastremain']) and ((remain - _nonlocal['lastremain'])/_nonlocal['lastremain'] < 0.05)): remain = _nonlocal['lastremain'] else: _nonlocal['lastremain'] = remain hours = int(remain/60/60) mins = int(remain/60 - hours*60) secs = int(remain - hours*60*60 - mins*60) if elapsed < 30: remainstr = '' kpsstr = '' else: kpsstr = '%dk/s' % kps if hours: remainstr = '%dh%dm' % (hours, mins) elif mins: remainstr = '%dm%d' % (mins, secs) else: remainstr = '%ds' % secs qprogress('Saving: %.2f%% (%d/%dk, %d/%d files) %s %s\r' % (pct, cc/1024, total/1024, fcount, ftotal, remainstr, kpsstr)) indexfile = opt.indexfile or git.repo(b'bupindex') r = index.Reader(indexfile) try: msr = index.MetaStoreReader(indexfile + b'.meta') except IOError as ex: if ex.errno != EACCES: raise log('error: cannot access %r; have you run bup index?' % path_msg(indexfile)) sys.exit(1) hlink_db = hlinkdb.HLinkDB(indexfile + b'.hlink') def already_saved(ent): return ent.is_valid() and w.exists(ent.sha) and ent.sha def wantrecurse_pre(ent): return not already_saved(ent) def wantrecurse_during(ent): return not already_saved(ent) or ent.sha_missing() def find_hardlink_target(hlink_db, ent): if hlink_db and not stat.S_ISDIR(ent.mode) and ent.nlink > 1: link_paths = hlink_db.node_paths(ent.dev, ent.ino) if link_paths: return link_paths[0] total = ftotal = 0 if opt.progress: for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_pre): if not (ftotal % 10024): qprogress('Reading index: %d\r' % ftotal) exists = ent.exists() hashvalid = already_saved(ent) ent.set_sha_missing(not hashvalid) if not opt.smaller or ent.size < opt.smaller: if exists and not hashvalid: total += ent.size ftotal += 1 progress('Reading index: %d, done.\n' % ftotal) hashsplit.progress_callback = progress_report # Root collisions occur when strip or graft options map more than one # path to the same directory (paths which originally had separate # parents). When that situation is detected, use empty metadata for # the parent. Otherwise, use the metadata for the common parent. # Collision example: "bup save ... --strip /foo /foo/bar /bar". # FIXME: Add collision tests, or handle collisions some other way. # FIXME: Detect/handle strip/graft name collisions (other than root), # i.e. if '/foo/bar' and '/bar' both map to '/'. first_root = None root_collision = None tstart = time.time() fcount = 0 lastskip_name = None lastdir = b'' for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during): (dir, file) = os.path.split(ent.name) exists = (ent.flags & index.IX_EXISTS) hashvalid = already_saved(ent) wasmissing = ent.sha_missing() oldsize = ent.size if opt.verbose: if not exists: status = 'D' elif not hashvalid: if ent.sha == index.EMPTY_SHA: status = 'A' else: status = 'M' else: status = ' ' if opt.verbose >= 2: log('%s %-70s\n' % (status, path_msg(ent.name))) elif not stat.S_ISDIR(ent.mode) and lastdir != dir: if not lastdir.startswith(dir): log('%s %-70s\n' % (status, path_msg(os.path.join(dir, b'')))) lastdir = dir if opt.progress: progress_report(0) fcount += 1 if not exists: continue if opt.smaller and ent.size >= opt.smaller: if exists and not hashvalid: if opt.verbose: log('skipping large file "%s"\n' % path_msg(ent.name)) lastskip_name = ent.name continue assert(dir.startswith(b'/')) if opt.strip: dirp = stripped_path_components(dir, extra) elif opt.strip_path: dirp = stripped_path_components(dir, [opt.strip_path]) elif graft_points: dirp = grafted_path_components(graft_points, dir) else: dirp = path_components(dir) # At this point, dirp contains a representation of the archive # path that looks like [(archive_dir_name, real_fs_path), ...]. # So given "bup save ... --strip /foo/bar /foo/bar/baz", dirp # might look like this at some point: # [('', '/foo/bar'), ('baz', '/foo/bar/baz'), ...]. # This dual representation supports stripping/grafting, where the # archive path may not have a direct correspondence with the # filesystem. The root directory is represented by an initial # component named '', and any component that doesn't have a # corresponding filesystem directory (due to grafting, for # example) will have a real_fs_path of None, i.e. [('', None), # ...]. if first_root == None: first_root = dirp[0] elif first_root != dirp[0]: root_collision = True # If switching to a new sub-tree, finish the current sub-tree. while [x.name for x in stack] > [x[0] for x in dirp]: _pop() # If switching to a new sub-tree, start a new sub-tree. for path_component in dirp[len(stack):]: dir_name, fs_path = path_component # Not indexed, so just grab the FS metadata or use empty metadata. try: meta = metadata.from_path(fs_path, normalized=True) \ if fs_path else metadata.Metadata() except (OSError, IOError) as e: add_error(e) lastskip_name = dir_name meta = metadata.Metadata() _push(dir_name, meta) if not file: if len(stack) == 1: continue # We're at the top level -- keep the current root dir # Since there's no filename, this is a subdir -- finish it. oldtree = already_saved(ent) # may be None newtree = _pop(force_tree = oldtree) if not oldtree: if lastskip_name and lastskip_name.startswith(ent.name): ent.invalidate() else: ent.validate(GIT_MODE_TREE, newtree) ent.repack() if exists and wasmissing: _nonlocal['count'] += oldsize continue # it's not a directory if hashvalid: meta = msr.metadata_at(ent.meta_ofs) meta.hardlink_target = find_hardlink_target(hlink_db, ent) # Restore the times that were cleared to 0 in the metastore. (meta.atime, meta.mtime, meta.ctime) = (ent.atime, ent.mtime, ent.ctime) stack[-1].append(file, ent.mode, ent.gitmode, ent.sha, meta) else: id = None hlink = find_hardlink_target(hlink_db, ent) try: meta = metadata.from_path(ent.name, hardlink_target=hlink, normalized=True, after_stat=after_nondir_metadata_stat) except (OSError, IOError) as e: add_error(e) lastskip_name = ent.name continue if stat.S_IFMT(ent.mode) != stat.S_IFMT(meta.mode): # The mode changed since we indexed the file, this is bad. # This can cause two issues: # 1) We e.g. think the file is a regular file, but now it's # something else (a device, socket, FIFO or symlink, etc.) # and _read_ from it when we shouldn't. # 2) We then record it as valid, but don't update the index # metadata, and on a subsequent save it has 'hashvalid' # but is recorded as the file type from the index, when # the content is something else ... # Avoid all of these consistency issues by just skipping such # things - it really ought to not happen anyway. add_error("%s: mode changed since indexing, skipping." % path_msg(ent.name)) lastskip_name = ent.name continue if stat.S_ISREG(ent.mode): try: # If the file changes while we're reading it, then our reading # may stop at some point, but the stat() above may have gotten # a different size already. Recalculate the meta size so that # the repository records the accurate size in the metadata, even # if the other stat() data might be slightly older than the file # content (which we can't fix, this is inherently racy, but we # can prevent the size mismatch.) meta.size = 0 def new_blob(data): meta.size += len(data) return w.new_blob(data) before_saving_regular_file(ent.name) with hashsplit.open_noatime(ent.name) as f: (mode, id) = hashsplit.split_to_blob_or_tree( new_blob, w.new_tree, [f], keep_boundaries=False) except (IOError, OSError) as e: add_error('%s: %s' % (ent.name, e)) lastskip_name = ent.name elif stat.S_ISDIR(ent.mode): assert(0) # handled above elif stat.S_ISLNK(ent.mode): mode, id = (GIT_MODE_SYMLINK, w.new_blob(meta.symlink_target)) else: # Everything else should be fully described by its # metadata, so just record an empty blob, so the paths # in the tree and .bupm will match up. (mode, id) = (GIT_MODE_FILE, w.new_blob(b'')) if id: ent.validate(mode, id) ent.repack() stack[-1].append(file, ent.mode, ent.gitmode, id, meta) if exists and wasmissing: _nonlocal['count'] += oldsize _nonlocal['subcount'] = 0 if opt.progress: pct = total and _nonlocal['count']*100.0/total or 100 progress('Saving: %.2f%% (%d/%dk, %d/%d files), done. \n' % (pct, _nonlocal['count']/1024, total/1024, fcount, ftotal)) while len(stack) > 1: # _pop() all the parts above the root _pop() # Finish the root directory. # When there's a collision, use empty metadata for the root. tree = _pop(dir_metadata = metadata.Metadata() if root_collision else None) sys.stdout.flush() out = byte_stream(sys.stdout) if opt.tree: out.write(hexlify(tree)) out.write(b'\n') if opt.commit or name: if compat.py_maj > 2: # Strip b prefix from python 3 bytes reprs to preserve previous format msgcmd = b'[%s]' % b', '.join([repr(argv_bytes(x))[1:].encode('ascii') for x in argv]) else: msgcmd = repr(argv) msg = b'bup save\n\nGenerated by command:\n%s\n' % msgcmd userline = (b'%s <%s@%s>' % (userfullname(), username(), hostname())) commit = w.new_commit(tree, oldref, userline, date, None, userline, date, None, msg) if opt.commit: out.write(hexlify(commit)) out.write(b'\n') msr.close() w.close() # must close before we can update the ref if opt.name: if cli: cli.update_ref(refname, commit, oldref) else: git.update_ref(refname, commit, oldref) if cli: cli.close() if saved_errors: log('WARNING: %d errors encountered while saving.\n' % len(saved_errors)) sys.exit(1)
n,name= name of backup set to update (if any) d,date= date for the commit (seconds since the epoch) v,verbose increase log output (can be used more than once) q,quiet don't show progress meter smaller= only back up files smaller than n bytes bwlimit= maximum bytes/sec to transmit to server f,indexfile= the name of the index file (normally BUP_DIR/bupindex) strip strips the path to every filename given strip-path= path-prefix to be stripped when saving graft= a graft point *old_path*=*new_path* (can be used more than once) #,compress= set compression level to # (0-9, 9 is highest) [1] """ o = options.Options(optspec) (opt, flags, extra) = o.parse(sys.argv[1:]) git.check_repo_or_die() if not (opt.tree or opt.commit or opt.name): o.fatal("use one or more of -t, -c, -n") if not extra: o.fatal("no filenames given") opt.progress = (istty2 and not opt.quiet) opt.smaller = parse_num(opt.smaller or 0) if opt.bwlimit: client.bwlimit = parse_num(opt.bwlimit) if opt.date: date = parse_date_or_fatal(opt.date, o.fatal) else: date = time.time()
def test_commit_parsing(): def restore_env_var(name, val): if val is None: del os.environ[name] else: os.environ[name] = val def showval(commit, val): return readpipe(['git', 'show', '-s', '--pretty=format:%s' % val, commit]).strip() with no_lingering_errors(), test_tempdir('bup-tgit-') as tmpdir: orig_cwd = os.getcwd() workdir = tmpdir + "/work" repodir = workdir + '/.git' orig_author_name = os.environ.get('GIT_AUTHOR_NAME') orig_author_email = os.environ.get('GIT_AUTHOR_EMAIL') orig_committer_name = os.environ.get('GIT_COMMITTER_NAME') orig_committer_email = os.environ.get('GIT_COMMITTER_EMAIL') os.environ['GIT_AUTHOR_NAME'] = 'bup test' os.environ['GIT_COMMITTER_NAME'] = os.environ['GIT_AUTHOR_NAME'] os.environ['GIT_AUTHOR_EMAIL'] = 'bup@a425bc70a02811e49bdf73ee56450e6f' os.environ['GIT_COMMITTER_EMAIL'] = os.environ['GIT_AUTHOR_EMAIL'] try: readpipe(['git', 'init', workdir]) os.environ['GIT_DIR'] = os.environ['BUP_DIR'] = repodir git.check_repo_or_die(repodir) os.chdir(workdir) with open('foo', 'w') as f: print >> f, 'bar' readpipe(['git', 'add', '.']) readpipe(['git', 'commit', '-am', 'Do something', '--author', 'Someone <someone@somewhere>', '--date', 'Sat Oct 3 19:48:49 2009 -0400']) commit = readpipe(['git', 'show-ref', '-s', 'master']).strip() parents = showval(commit, '%P') tree = showval(commit, '%T') cname = showval(commit, '%cn') cmail = showval(commit, '%ce') cdate = showval(commit, '%ct') coffs = showval(commit, '%ci') coffs = coffs[-5:] coff = (int(coffs[-4:-2]) * 60 * 60) + (int(coffs[-2:]) * 60) if coffs[-5] == '-': coff = - coff commit_items = git.get_commit_items(commit, git.cp()) WVPASSEQ(commit_items.parents, []) WVPASSEQ(commit_items.tree, tree) WVPASSEQ(commit_items.author_name, 'Someone') WVPASSEQ(commit_items.author_mail, 'someone@somewhere') WVPASSEQ(commit_items.author_sec, 1254613729) WVPASSEQ(commit_items.author_offset, -(4 * 60 * 60)) WVPASSEQ(commit_items.committer_name, cname) WVPASSEQ(commit_items.committer_mail, cmail) WVPASSEQ(commit_items.committer_sec, int(cdate)) WVPASSEQ(commit_items.committer_offset, coff) WVPASSEQ(commit_items.message, 'Do something\n') with open('bar', 'w') as f: print >> f, 'baz' readpipe(['git', 'add', '.']) readpipe(['git', 'commit', '-am', 'Do something else']) child = readpipe(['git', 'show-ref', '-s', 'master']).strip() parents = showval(child, '%P') commit_items = git.get_commit_items(child, git.cp()) WVPASSEQ(commit_items.parents, [commit]) finally: os.chdir(orig_cwd) restore_env_var('GIT_AUTHOR_NAME', orig_author_name) restore_env_var('GIT_AUTHOR_EMAIL', orig_author_email) restore_env_var('GIT_COMMITTER_NAME', orig_committer_name) restore_env_var('GIT_COMMITTER_EMAIL', orig_committer_email)
def receive_objects(conn, junk): global suspended_w git.check_repo_or_die() suggested = {} if suspended_w: w = suspended_w suspended_w = None else: w = git.PackWriter() while 1: ns = conn.read(4) if not ns: w.abort() raise Exception('object read: expected length header, got EOF\n') n = struct.unpack('!I', ns)[0] #log('expecting %d bytes\n' % n) if not n: log('bup server: received %d object%s.\n' % (w.count, w.count!=1 and "s" or '')) fullpath = w.close() if fullpath: (dir, name) = os.path.split(fullpath) conn.write('%s.idx\n' % name) conn.ok() return elif n == 0xffffffff: log('bup server: receive-objects suspended.\n') suspended_w = w conn.ok() return buf = conn.read(n) # object sizes in bup are reasonably small #log('read %d bytes\n' % n) if len(buf) < n: w.abort() raise Exception('object read: expected %d bytes, got %d\n' % (n, len(buf))) (type, content) = git._decode_packobj(buf) sha = git.calc_hash(type, content) oldpack = w.exists(sha) # FIXME: we only suggest a single index per cycle, because the client # is currently dumb to download more than one per cycle anyway. # Actually we should fix the client, but this is a minor optimization # on the server side. if not suggested and \ oldpack and (oldpack == True or oldpack.endswith('.midx')): # FIXME: we shouldn't really have to know about midx files # at this layer. But exists() on a midx doesn't return the # packname (since it doesn't know)... probably we should just # fix that deficiency of midx files eventually, although it'll # make the files bigger. This method is certainly not very # efficient. w.objcache.refresh(skip_midx = True) oldpack = w.objcache.exists(sha) log('new suggestion: %r\n' % oldpack) assert(oldpack) assert(oldpack != True) assert(not oldpack.endswith('.midx')) w.objcache.refresh(skip_midx = False) if not suggested and oldpack: assert(oldpack.endswith('.idx')) (dir,name) = os.path.split(oldpack) if not (name in suggested): log("bup server: suggesting index %s\n" % name) conn.write('index %s\n' % name) suggested[name] = 1 else: w._raw_write([buf])
def main(argv): o = options.Options(optspec) opt, flags, roots = o.parse_bytes(argv[1:]) roots = [argv_bytes(x) for x in roots] if not opt.unsafe: o.fatal( 'refusing to run dangerous, experimental command without --unsafe') now = int(time()) if opt.wrt is None else opt.wrt if not isinstance(now, int): o.fatal('--wrt value ' + str(now) + ' is not an integer') period_start = {} for period, extent in (('all', opt.keep_all_for), ('dailies', opt.keep_dailies_for), ('monthlies', opt.keep_monthlies_for), ('yearlies', opt.keep_yearlies_for)): if extent: secs = period_as_secs(extent.encode('ascii')) if not secs: o.fatal('%r is not a valid period' % extent) period_start[period] = now - secs if not period_start: o.fatal('at least one keep argument is required') period_start = defaultdict(lambda: float('inf'), period_start) if opt.verbose: epoch_ymd = strftime('%Y-%m-%d-%H%M%S', localtime(0)) for kind in ['all', 'dailies', 'monthlies', 'yearlies']: period_utc = period_start[kind] if period_utc != float('inf'): if not (period_utc > float('-inf')): log('keeping all ' + kind) else: try: when = strftime('%Y-%m-%d-%H%M%S', localtime(period_utc)) log('keeping ' + kind + ' since ' + when + '\n') except ValueError as ex: if period_utc < 0: log('keeping %s since %d seconds before %s\n' % (kind, abs(period_utc), epoch_ymd)) elif period_utc > 0: log('keeping %s since %d seconds after %s\n' % (kind, period_utc, epoch_ymd)) else: log('keeping %s since %s\n' % (kind, epoch_ymd)) git.check_repo_or_die() # This could be more efficient, but for now just build the whole list # in memory and let bup_rm() do some redundant work. def parse_info(f): author_secs = f.readline().strip() return int(author_secs) sys.stdout.flush() out = byte_stream(sys.stdout) removals = [] for branch, branch_id in branches(roots): die_if_errors() saves = ((utc, unhexlify(oidx)) for ( oidx, utc) in git.rev_list(branch_id, format=b'%at', parse=parse_info)) for keep_save, (utc, id) in classify_saves(saves, period_start): assert (keep_save in (False, True)) # FIXME: base removals on hashes if opt.pretend: out.write((b'+ ' if keep_save else b'- ') + save_name(branch, utc) + b'\n') elif not keep_save: removals.append(save_name(branch, utc)) if not opt.pretend: die_if_errors() with LocalRepo() as repo: bup_rm(repo, removals, compression=opt.compress, verbosity=opt.verbose) if opt.gc: die_if_errors() bup_gc(threshold=opt.gc_threshold, compression=opt.compress, verbosity=opt.verbose) die_if_errors()
handle_ctrl_c() optspec = """ bup tag bup tag [-f] <tag name> <commit> bup tag [-f] -d <tag name> -- d,delete= Delete a tag f,force Overwrite existing tag, or ignore missing tag when deleting """ o = options.Options(optspec) (opt, flags, extra) = o.parse(sys.argv[1:]) git.check_repo_or_die() tags = [t for sublist in git.tags().values() for t in sublist] if opt.delete: # git.delete_ref() doesn't complain if a ref doesn't exist. We # could implement this verification but we'd need to read in the # contents of the tag file and pass the hash, and we already know # about the tag's existance via "tags". if not opt.force and opt.delete not in tags: log("error: tag '%s' doesn't exist\n" % opt.delete) sys.exit(1) tag_file = 'refs/tags/%s' % opt.delete git.delete_ref(tag_file) sys.exit(0)
def main(argv): o = options.Options(optspec) opt, flags, extra = o.parse_bytes(argv[1:]) if opt.name: opt.name = argv_bytes(opt.name) if opt.remote: opt.remote = argv_bytes(opt.remote) if opt.verbose is None: opt.verbose = 0 if not (opt.blobs or opt.tree or opt.commit or opt.name or opt.noop or opt.copy): o.fatal("use one or more of -b, -t, -c, -n, --noop, --copy") if opt.copy and (opt.blobs or opt.tree): o.fatal('--copy is incompatible with -b, -t') if (opt.noop or opt.copy) and (opt.commit or opt.name): o.fatal('--noop and --copy are incompatible with -c, -n') if opt.blobs and (opt.tree or opt.commit or opt.name): o.fatal('-b is incompatible with -t, -c, -n') if extra and opt.git_ids: o.fatal("don't provide filenames when using --git-ids") if opt.verbose >= 2: git.verbose = opt.verbose - 1 opt.bench = 1 max_pack_size = None if opt.max_pack_size: max_pack_size = parse_num(opt.max_pack_size) max_pack_objects = None if opt.max_pack_objects: max_pack_objects = parse_num(opt.max_pack_objects) if opt.fanout: hashsplit.fanout = parse_num(opt.fanout) if opt.blobs: hashsplit.fanout = 0 if opt.bwlimit: client.bwlimit = parse_num(opt.bwlimit) if opt.date: date = parse_date_or_fatal(opt.date, o.fatal) else: date = time.time() # Hack around lack of nonlocal vars in python 2 total_bytes = [0] def prog(filenum, nbytes): total_bytes[0] += nbytes if filenum > 0: qprogress('Splitting: file #%d, %d kbytes\r' % (filenum + 1, total_bytes[0] // 1024)) else: qprogress('Splitting: %d kbytes\r' % (total_bytes[0] // 1024)) is_reverse = environ.get(b'BUP_SERVER_REVERSE') if is_reverse and opt.remote: o.fatal("don't use -r in reverse mode; it's automatic") start_time = time.time() if opt.name and not valid_save_name(opt.name): o.fatal("'%r' is not a valid branch name." % opt.name) refname = opt.name and b'refs/heads/%s' % opt.name or None if opt.noop or opt.copy: cli = pack_writer = oldref = None elif opt.remote or is_reverse: git.check_repo_or_die() cli = client.Client(opt.remote) oldref = refname and cli.read_ref(refname) or None pack_writer = cli.new_packwriter(compression_level=opt.compress, max_pack_size=max_pack_size, max_pack_objects=max_pack_objects) else: git.check_repo_or_die() cli = None oldref = refname and git.read_ref(refname) or None pack_writer = git.PackWriter(compression_level=opt.compress, max_pack_size=max_pack_size, max_pack_objects=max_pack_objects) input = byte_stream(sys.stdin) if opt.git_ids: # the input is actually a series of git object ids that we should retrieve # and split. # # This is a bit messy, but basically it converts from a series of # CatPipe.get() iterators into a series of file-type objects. # It would be less ugly if either CatPipe.get() returned a file-like object # (not very efficient), or split_to_shalist() expected an iterator instead # of a file. cp = git.CatPipe() class IterToFile: def __init__(self, it): self.it = iter(it) def read(self, size): v = next(self.it, None) return v or b'' def read_ids(): while 1: line = input.readline() if not line: break if line: line = line.strip() try: it = cp.get(line.strip()) next(it, None) # skip the file info except KeyError as e: add_error('error: %s' % e) continue yield IterToFile(it) files = read_ids() else: # the input either comes from a series of files or from stdin. files = extra and (open(argv_bytes(fn), 'rb') for fn in extra) or [input] if pack_writer: new_blob = pack_writer.new_blob new_tree = pack_writer.new_tree elif opt.blobs or opt.tree: # --noop mode new_blob = lambda content: git.calc_hash(b'blob', content) new_tree = lambda shalist: git.calc_hash(b'tree', git.tree_encode(shalist)) sys.stdout.flush() out = byte_stream(sys.stdout) if opt.blobs: shalist = hashsplit.split_to_blobs(new_blob, files, keep_boundaries=opt.keep_boundaries, progress=prog) for (sha, size, level) in shalist: out.write(hexlify(sha) + b'\n') reprogress() elif opt.tree or opt.commit or opt.name: if opt.name: # insert dummy_name which may be used as a restore target mode, sha = \ hashsplit.split_to_blob_or_tree(new_blob, new_tree, files, keep_boundaries=opt.keep_boundaries, progress=prog) splitfile_name = git.mangle_name(b'data', hashsplit.GIT_MODE_FILE, mode) shalist = [(mode, splitfile_name, sha)] else: shalist = hashsplit.split_to_shalist( new_blob, new_tree, files, keep_boundaries=opt.keep_boundaries, progress=prog) tree = new_tree(shalist) else: last = 0 it = hashsplit.hashsplit_iter(files, keep_boundaries=opt.keep_boundaries, progress=prog) for (blob, level) in it: hashsplit.total_split += len(blob) if opt.copy: sys.stdout.write(str(blob)) megs = hashsplit.total_split // 1024 // 1024 if not opt.quiet and last != megs: last = megs if opt.verbose: log('\n') if opt.tree: out.write(hexlify(tree) + b'\n') if opt.commit or opt.name: msg = b'bup split\n\nGenerated by command:\n%r\n' % compat.get_argvb() ref = opt.name and (b'refs/heads/%s' % opt.name) or None userline = b'%s <%s@%s>' % (userfullname(), username(), hostname()) commit = pack_writer.new_commit(tree, oldref, userline, date, None, userline, date, None, msg) if opt.commit: out.write(hexlify(commit) + b'\n') if pack_writer: pack_writer.close() # must close before we can update the ref if opt.name: if cli: cli.update_ref(refname, commit, oldref) else: git.update_ref(refname, commit, oldref) if cli: cli.close() secs = time.time() - start_time size = hashsplit.total_split if opt.bench: log('bup: %.2f kbytes in %.2f secs = %.2f kbytes/sec\n' % (size / 1024, secs, size / 1024 / secs)) if saved_errors: log('WARNING: %d errors encountered while saving.\n' % len(saved_errors)) sys.exit(1)