def main(argv): o = options.Options(optspec) opt, flags, extra = o.parse_bytes(argv[1:]) stdin = byte_stream(sys.stdin) if not extra: extra = linereader(stdin) ret = 0 repo = from_opts(opt) if opt.o: outfile = open(opt.o, 'wb') else: sys.stdout.flush() outfile = byte_stream(sys.stdout) for ref in [argv_bytes(x) for x in extra]: try: for blob in repo.join(ref): outfile.write(blob) except KeyError as e: outfile.flush() log('error: %s\n' % e) ret = 1 sys.exit(ret)
def via_cmdline(args, out=None, onabort=None): """Write a listing of a file or directory in the bup repository to out. When a long listing is not requested and stdout is attached to a tty, the output is formatted in columns. When not attached to tty (for example when the output is piped to another command), one file is listed per line. """ assert out opt = opts_from_cmdline(args, onabort=onabort) r = repo.from_opts(opt, reverse=False) return within_repo(r, opt, out)
def main(argv): o = options.Options(optspec) opt, flags, extra = o.parse_bytes(argv[1:]) if not extra: o.fatal('must specify a target') if len(extra) > 1: o.fatal('only one target file allowed') if opt.bupm and opt.meta: o.fatal('--meta and --bupm are incompatible') target = argv_bytes(extra[0]) if not re.match(br'/*[^/]+/[^/]+', target): o.fatal("path %r doesn't include a branch and revision" % target) repo = from_opts(opt, reverse=False) resolved = vfs.resolve(repo, target, follow=False) leaf_name, leaf_item = resolved[-1] if not leaf_item: log('error: cannot access %r in %r\n' % (b'/'.join(name for name, item in resolved), target)) sys.exit(1) mode = vfs.item_mode(leaf_item) sys.stdout.flush() out = byte_stream(sys.stdout) if opt.bupm: if not stat.S_ISDIR(mode): o.fatal('%r is not a directory' % target) _, bupm_oid = vfs.tree_data_and_bupm(repo, leaf_item.oid) if bupm_oid: with vfs.tree_data_reader(repo, bupm_oid) as meta_stream: out.write(meta_stream.read()) elif opt.meta: augmented = vfs.augment_item_meta(repo, leaf_item, include_size=True) out.write(augmented.meta.encode()) else: if stat.S_ISREG(mode): with vfs.fopen(repo, leaf_item) as f: for b in chunkyreader(f): out.write(b) else: o.fatal('%r is not a plain file' % target) if saved_errors: log('warning: %d errors encountered\n' % len(saved_errors)) sys.exit(1)
def main(argv): o = Options(optspec) opt, flags, extra = o.parse_bytes(argv[1:]) if not opt.unsafe: o.fatal( 'refusing to run dangerous, experimental command without --unsafe') if len(extra) < 1: o.fatal('no paths specified') repo = from_opts(opt) bup_rm(repo, [argv_bytes(x) for x in extra], verbosity=opt.verbose) die_if_errors()
def main(argv): o = options.Options(optspec) (opt, flags, extra) = o.parse(argv[1:]) if len(extra) != 1: o.fatal("must give exactly one name") name = argv_bytes(extra[0]) r = repo.from_opts(opt) if opt.type == 'str': opt.type = None print("%s = %r" % (name.decode('utf-8'), r.config(name, opttype=opt.type))) r.close()
parameter = argv_bytes(parameter) splitted_parameter = parameter.split(b'=') if len(splitted_parameter) != 2: o.fatal("a graft point must be of the form old_path=new_path") old_path, new_path = splitted_parameter if not (old_path and new_path): o.fatal("a graft point cannot be empty") graft_points.append( (resolve_parent(old_path), resolve_parent(new_path))) name = opt.name if name and not valid_save_name(name): o.fatal("'%s' is not a valid branch name" % path_msg(name)) refname = name and b'refs/heads/%s' % name or None repo = repo.from_opts(opt) use_treesplit = repo.config(b'bup.treesplit', opttype='bool') blobbits = repo.config(b'bup.blobbits', opttype='int') oldref = refname and repo.read_ref(refname) or None handle_ctrl_c() # Metadata is stored in a file named .bupm in each directory. The # first metadata entry will be the metadata for the current directory. # The remaining entries will be for each of the other directory # elements, in the order they're listed in the index. # # Since the git tree elements are sorted according to # git.shalist_item_sort_key, the metalist items are accumulated as # (sort_key, metadata) tuples, and then sorted when the .bupm file is
def main(argv): o = options.Options(optspec) opt, flags, roots = o.parse_bytes(argv[1:]) roots = [argv_bytes(x) for x in roots] if not opt.unsafe: o.fatal( 'refusing to run dangerous, experimental command without --unsafe') now = int(time()) if opt.wrt is None else opt.wrt if not isinstance(now, int_types): o.fatal('--wrt value ' + str(now) + ' is not an integer') period_start = {} for period, extent in (('all', opt.keep_all_for), ('dailies', opt.keep_dailies_for), ('monthlies', opt.keep_monthlies_for), ('yearlies', opt.keep_yearlies_for)): if extent: secs = period_as_secs(extent.encode('ascii')) if not secs: o.fatal('%r is not a valid period' % extent) period_start[period] = now - secs if not period_start: o.fatal('at least one keep argument is required') period_start = defaultdict(lambda: float('inf'), period_start) if opt.verbose: epoch_ymd = strftime('%Y-%m-%d-%H%M%S', localtime(0)) for kind in ['all', 'dailies', 'monthlies', 'yearlies']: period_utc = period_start[kind] if period_utc != float('inf'): if not (period_utc > float('-inf')): log('keeping all ' + kind) else: try: when = strftime('%Y-%m-%d-%H%M%S', localtime(period_utc)) log('keeping ' + kind + ' since ' + when + '\n') except ValueError as ex: if period_utc < 0: log('keeping %s since %d seconds before %s\n' % (kind, abs(period_utc), epoch_ymd)) elif period_utc > 0: log('keeping %s since %d seconds after %s\n' % (kind, period_utc, epoch_ymd)) else: log('keeping %s since %s\n' % (kind, epoch_ymd)) # This could be more efficient, but for now just build the whole list # in memory and let bup_rm() do some redundant work. def parse_info(f): author_secs = f.readline().strip() return int(author_secs) sys.stdout.flush() out = byte_stream(sys.stdout) repo = from_opts(opt) removals = [] for branch, branch_id in branches(repo, roots): die_if_errors() saves = ((utc, unhexlify(oidx)) for ( oidx, utc) in repo.rev_list(branch_id, format=b'%at', parse=parse_info)) for keep_save, (utc, id) in classify_saves(saves, period_start): assert (keep_save in (False, True)) # FIXME: base removals on hashes if opt.pretend: out.write((b'+ ' if keep_save else b'- ') + save_name(branch, utc) + b'\n') elif not keep_save: removals.append(save_name(branch, utc)) if not opt.pretend: die_if_errors() bup_rm(repo, removals, verbosity=opt.verbose) if opt.gc: die_if_errors() bup_gc(repo, threshold=opt.gc_threshold, compression=opt.compress, verbosity=opt.verbose) die_if_errors()
def main(): o = options.Options(optspec) opt, flags, extra = o.parse(sys.argv[1:]) verbosity = (opt.verbose or 0) if not opt.quiet else -1 if opt.remote: opt.remote = argv_bytes(opt.remote) if opt.outdir: opt.outdir = argv_bytes(opt.outdir) git.check_repo_or_die() if not extra: o.fatal('must specify at least one filename to restore') exclude_rxs = parse_rx_excludes(flags, o.fatal) owner_map = {} for map_type in ('user', 'group', 'uid', 'gid'): owner_map[map_type] = parse_owner_mappings(map_type, flags, o.fatal) if opt.outdir: mkdirp(opt.outdir) os.chdir(opt.outdir) if opt.remote: opt.remote = argv_bytes(opt.remote) repo = from_opts(opt, reverse=False) top = fsencode(os.getcwd()) hardlinks = {} for path in [argv_bytes(x) for x in extra]: if not valid_restore_path(path): add_error("path %r doesn't include a branch and revision" % path) continue try: resolved = vfs.resolve(repo, path, want_meta=True, follow=False) except vfs.IOError as e: add_error(e) continue if len(resolved) == 3 and resolved[2][0] == b'latest': # Follow latest symlink to the actual save try: resolved = vfs.resolve(repo, b'latest', parent=resolved[:-1], want_meta=True) except vfs.IOError as e: add_error(e) continue # Rename it back to 'latest' resolved = tuple(elt if i != 2 else (b'latest', ) + elt[1:] for i, elt in enumerate(resolved)) path_parent, path_name = os.path.split(path) leaf_name, leaf_item = resolved[-1] if not leaf_item: add_error('error: cannot access %r in %r' % ('/'.join(name for name, item in resolved), path)) continue if not path_name or path_name == b'.': # Source is /foo/what/ever/ or /foo/what/ever/. -- extract # what/ever/* to the current directory, and if name == '.' # (i.e. /foo/what/ever/.), then also restore what/ever's # metadata to the current directory. treeish = vfs.item_mode(leaf_item) if not treeish: add_error('%r cannot be restored as a directory' % path) else: items = vfs.contents(repo, leaf_item, want_meta=True) dot, leaf_item = next(items, None) assert dot == b'.' for sub_name, sub_item in items: restore(repo, b'', sub_name, sub_item, top, opt.sparse, opt.numeric_ids, owner_map, exclude_rxs, verbosity, hardlinks) if path_name == b'.': leaf_item = vfs.augment_item_meta(repo, leaf_item, include_size=True) apply_metadata(leaf_item.meta, b'.', opt.numeric_ids, owner_map) else: restore(repo, b'', leaf_name, leaf_item, top, opt.sparse, opt.numeric_ids, owner_map, exclude_rxs, verbosity, hardlinks) if verbosity >= 0: progress('Restoring: %d, done.\n' % total_restored) die_if_errors()
settings = dict( debug=1, template_path=resource_path(b'web').decode('utf-8'), static_path=resource_path(b'web/static').decode('utf-8'), ) # Disable buffering on stdout, for debug messages try: sys.stdout._line_buffering = True except AttributeError: sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) if opt.remote: opt.remote = argv_bytes(opt.remote) repo = from_opts(opt, reverse=False) application = tornado.web.Application([ (r"(?P<path>/.*)", BupRequestHandler, dict(repo=repo)), ], **settings) http_server = HTTPServer(application) io_loop_pending = IOLoop.instance() if isinstance(address, InetAddress): sockets = tornado.netutil.bind_sockets(address.port, address.host) http_server.add_sockets(sockets) print('Serving HTTP on %s:%d...' % sockets[0].getsockname()) if opt.browser: browser_addr = 'http://' + address[0] + ':' + str(address[1]) io_loop_pending.add_callback(lambda: webbrowser.open(browser_addr))
def main(argv): o = options.Options(optspec) opt, flags, extra = o.parse_bytes(argv[1:]) global repo repo = from_opts(opt, reverse=False) sys.stdout.flush() out = byte_stream(sys.stdout) stdin = byte_stream(sys.stdin) pwd = vfs.resolve(repo, b'/') rv = 0 def inputiter(f): if os.isatty(f.fileno()): while 1: prompt = b'bup %s> ' % (b'/'.join(name for name, item in pwd) or b'/', ) if hasattr(_helpers, 'readline'): try: yield _helpers.readline(prompt) except EOFError: print() # Clear the line for the terminal's next prompt break else: out.write(prompt) out.flush() read_line = f.readline() if not read_line: print('') break yield read_line else: for line in f: yield line if extra: lines = (argv_bytes(arg) for arg in extra) else: if hasattr(_helpers, 'readline'): _helpers.set_completer_word_break_characters(b' \t\n\r/') _helpers.set_attempted_completion_function(attempt_completion) _helpers.set_completion_entry_function(enter_completion) if sys.platform.startswith('darwin'): # MacOS uses a slightly incompatible clone of libreadline _helpers.parse_and_bind(b'bind ^I rl_complete') _helpers.parse_and_bind(b'tab: complete') lines = inputiter(stdin) for line in lines: if not line.strip(): continue words = [word for (wordstart,word) in shquote.quotesplit(line)] cmd = words[0].lower() #log('execute: %r %r\n' % (cmd, parm)) try: if cmd == b'ls': do_ls(repo, pwd, words[1:], out) out.flush() elif cmd == b'cd': np = pwd for parm in words[1:]: res = vfs.resolve(repo, parm, parent=np) _, leaf_item = res[-1] if not leaf_item: raise CommandError(b'"%s" does not exist' % b'/'.join(name for name, item in res)) if not stat.S_ISDIR(vfs.item_mode(leaf_item)): raise CommandError(b'"%s" is not a directory' % parm) np = res pwd = np elif cmd == b'pwd': if len(pwd) == 1: out.write(b'/') out.write(b'/'.join(name for name, item in pwd) + b'\n') out.flush() elif cmd == b'cat': for parm in words[1:]: res = vfs.resolve(repo, parm, parent=pwd) _, leaf_item = res[-1] if not leaf_item: raise CommandError(b'"%s" does not exist' % b'/'.join(name for name, item in res)) with vfs.fopen(repo, leaf_item) as srcfile: write_to_file(srcfile, out) out.flush() elif cmd == b'get': if len(words) not in [2,3]: rv = 1 raise CommandError(b'Usage: get <filename> [localname]') rname = words[1] (dir,base) = os.path.split(rname) lname = len(words) > 2 and words[2] or base res = vfs.resolve(repo, rname, parent=pwd) _, leaf_item = res[-1] if not leaf_item: raise CommandError(b'"%s" does not exist' % b'/'.join(name for name, item in res)) with vfs.fopen(repo, leaf_item) as srcfile: with open(lname, 'wb') as destfile: log('Saving %s\n' % path_msg(lname)) write_to_file(srcfile, destfile) elif cmd == b'mget': for parm in words[1:]: dir, base = os.path.split(parm) res = vfs.resolve(repo, dir, parent=pwd) _, dir_item = res[-1] if not dir_item: raise CommandError(b'"%s" does not exist' % dir) for name, item in vfs.contents(repo, dir_item): if name == b'.': continue if fnmatch.fnmatch(name, base): if stat.S_ISLNK(vfs.item_mode(item)): deref = vfs.resolve(repo, name, parent=res) deref_name, deref_item = deref[-1] if not deref_item: raise CommandError(b'"%s" does not exist' % b'/'.join(name for name, item in res)) item = deref_item with vfs.fopen(repo, item) as srcfile: with open(name, 'wb') as destfile: log('Saving %s\n' % path_msg(name)) write_to_file(srcfile, destfile) elif cmd == b'help' or cmd == b'?': out.write(b'Commands: ls cd pwd cat get mget help quit\n') out.flush() elif cmd in (b'quit', b'exit', b'bye'): break else: rv = 1 raise CommandError(b'no such command "%s"' % cmd) except CommandError as e: rv = 1 out.write(b'error: %s\n' % e.args[0]) out.flush() except Exception as e: rv = 1 out.write(b'error: %s\n' % str(e).encode()) out.flush() sys.exit(rv)
def main(argv): # Hack around lack of nonlocal vars in python 2 _nonlocal = {} o = options.Options(optspec) opt, flags, extra = o.parse_bytes(argv[1:]) if opt.indexfile: opt.indexfile = argv_bytes(opt.indexfile) if opt.name: opt.name = argv_bytes(opt.name) if opt.strip_path: opt.strip_path = argv_bytes(opt.strip_path) if not (opt.tree or opt.commit or opt.name): o.fatal("use one or more of -t, -c, -n") if not extra: o.fatal("no filenames given") extra = [argv_bytes(x) for x in extra] opt.progress = (istty2 and not opt.quiet) opt.smaller = parse_num(opt.smaller or 0) if opt.bwlimit: client.bwlimit = parse_num(opt.bwlimit) if opt.date: date = parse_date_or_fatal(opt.date, o.fatal) else: date = time.time() if opt.strip and opt.strip_path: o.fatal("--strip is incompatible with --strip-path") graft_points = [] if opt.graft: if opt.strip: o.fatal("--strip is incompatible with --graft") if opt.strip_path: o.fatal("--strip-path is incompatible with --graft") for (option, parameter) in flags: if option == "--graft": parameter = argv_bytes(parameter) splitted_parameter = parameter.split(b'=') if len(splitted_parameter) != 2: o.fatal( "a graft point must be of the form old_path=new_path") old_path, new_path = splitted_parameter if not (old_path and new_path): o.fatal("a graft point cannot be empty") graft_points.append( (resolve_parent(old_path), resolve_parent(new_path))) name = opt.name if name and not valid_save_name(name): o.fatal("'%s' is not a valid branch name" % path_msg(name)) refname = name and b'refs/heads/%s' % name or None repo = from_opts(opt) blobbits = repo.config(b'bup.blobbits', opttype='int') oldref = refname and repo.read_ref(refname) or None handle_ctrl_c() # Metadata is stored in a file named .bupm in each directory. The # first metadata entry will be the metadata for the current directory. # The remaining entries will be for each of the other directory # elements, in the order they're listed in the index. # # Since the git tree elements are sorted according to # git.shalist_item_sort_key, the metalist items are accumulated as # (sort_key, metadata) tuples, and then sorted when the .bupm file is # created. The sort_key should have been computed using the element's # mangled name and git mode (after hashsplitting), but the code isn't # actually doing that but rather uses the element's real name and mode. # This makes things a bit more difficult when reading it back, see # vfs.ordered_tree_entries(). # Maintain a stack of information representing the current location in # the archive being constructed. stack = Stack(repo) _nonlocal['count'] = 0 _nonlocal['subcount'] = 0 _nonlocal['lastremain'] = None def progress_report(file, n): _nonlocal['subcount'] += n cc = _nonlocal['count'] + _nonlocal['subcount'] pct = total and (cc * 100.0 / total) or 0 now = time.time() elapsed = now - tstart kps = elapsed and int(cc / 1024. / elapsed) kps_frac = 10**int(math.log(kps + 1, 10) - 1) kps = int(kps / kps_frac) * kps_frac if cc: remain = elapsed * 1.0 / cc * (total - cc) else: remain = 0.0 if (_nonlocal['lastremain'] and (remain > _nonlocal['lastremain']) and ((remain - _nonlocal['lastremain']) / _nonlocal['lastremain'] < 0.05)): remain = _nonlocal['lastremain'] else: _nonlocal['lastremain'] = remain hours = int(remain / 60 / 60) mins = int(remain / 60 - hours * 60) secs = int(remain - hours * 60 * 60 - mins * 60) if elapsed < 30: remainstr = '' kpsstr = '' else: kpsstr = '%dk/s' % kps if hours: remainstr = '%dh%dm' % (hours, mins) elif mins: remainstr = '%dm%d' % (mins, secs) else: remainstr = '%ds' % secs qprogress( 'Saving: %.2f%% (%d/%dk, %d/%d files) %s %s\r' % (pct, cc / 1024, total / 1024, fcount, ftotal, remainstr, kpsstr)) indexfile = opt.indexfile or path.index() r = index.Reader(indexfile) try: msr = index.MetaStoreReader(indexfile + b'.meta') except IOError as ex: if ex.errno != EACCES: raise log('error: cannot access %r; have you run bup index?' % path_msg(indexfile)) sys.exit(1) hlink_db = hlinkdb.HLinkDB(indexfile + b'.hlink') def already_saved(ent): return ent.is_valid() and repo.exists(ent.sha) and ent.sha def wantrecurse_pre(ent): return not already_saved(ent) def wantrecurse_during(ent): return not already_saved(ent) or ent.sha_missing() def find_hardlink_target(hlink_db, ent): if hlink_db and not stat.S_ISDIR(ent.mode) and ent.nlink > 1: link_paths = hlink_db.node_paths(ent.dev, ent.ino) if link_paths: return link_paths[0] total = ftotal = 0 if opt.progress: for (transname, ent) in r.filter(extra, wantrecurse=wantrecurse_pre): if not (ftotal % 10024): qprogress('Reading index: %d\r' % ftotal) exists = ent.exists() hashvalid = already_saved(ent) ent.set_sha_missing(not hashvalid) if not opt.smaller or ent.size < opt.smaller: if exists and not hashvalid: total += ent.size ftotal += 1 progress('Reading index: %d, done.\n' % ftotal) # Root collisions occur when strip or graft options map more than one # path to the same directory (paths which originally had separate # parents). When that situation is detected, use empty metadata for # the parent. Otherwise, use the metadata for the common parent. # Collision example: "bup save ... --strip /foo /foo/bar /bar". # FIXME: Add collision tests, or handle collisions some other way. # FIXME: Detect/handle strip/graft name collisions (other than root), # i.e. if '/foo/bar' and '/bar' both map to '/'. first_root = None root_collision = None tstart = time.time() fcount = 0 lastskip_name = None lastdir = b'' for (transname, ent) in r.filter(extra, wantrecurse=wantrecurse_during): (dir, file) = os.path.split(ent.name) exists = (ent.flags & index.IX_EXISTS) hashvalid = already_saved(ent) wasmissing = ent.sha_missing() oldsize = ent.size if opt.verbose: if not exists: status = 'D' elif not hashvalid: if ent.sha == index.EMPTY_SHA: status = 'A' else: status = 'M' else: status = ' ' if opt.verbose >= 2: log('%s %-70s\n' % (status, path_msg(ent.name))) elif not stat.S_ISDIR(ent.mode) and lastdir != dir: if not lastdir.startswith(dir): log('%s %-70s\n' % (status, path_msg(os.path.join(dir, b'')))) lastdir = dir if not opt.progress: progress_report = None fcount += 1 if not exists: continue if opt.smaller and ent.size >= opt.smaller: if exists and not hashvalid: if opt.verbose: log('skipping large file "%s"\n' % path_msg(ent.name)) lastskip_name = ent.name continue assert (dir.startswith(b'/')) if opt.strip: dirp = stripped_path_components(dir, extra) elif opt.strip_path: dirp = stripped_path_components(dir, [opt.strip_path]) elif graft_points: dirp = grafted_path_components(graft_points, dir) else: dirp = path_components(dir) # At this point, dirp contains a representation of the archive # path that looks like [(archive_dir_name, real_fs_path), ...]. # So given "bup save ... --strip /foo/bar /foo/bar/baz", dirp # might look like this at some point: # [('', '/foo/bar'), ('baz', '/foo/bar/baz'), ...]. # This dual representation supports stripping/grafting, where the # archive path may not have a direct correspondence with the # filesystem. The root directory is represented by an initial # component named '', and any component that doesn't have a # corresponding filesystem directory (due to grafting, for # example) will have a real_fs_path of None, i.e. [('', None), # ...]. if first_root == None: first_root = dirp[0] elif first_root != dirp[0]: root_collision = True # If switching to a new sub-tree, finish the current sub-tree. while list(stack.namestack) > [x[0] for x in dirp]: stack, _ = stack.pop() # If switching to a new sub-tree, start a new sub-tree. for path_component in dirp[len(stack):]: dir_name, fs_path = path_component # Not indexed, so just grab the FS metadata or use empty metadata. try: meta = metadata.from_path(fs_path, normalized=True) \ if fs_path else metadata.Metadata() except (OSError, IOError) as e: add_error(e) lastskip_name = dir_name meta = metadata.Metadata() stack = stack.push(dir_name, meta) if not file: if len(stack) == 1: continue # We're at the top level -- keep the current root dir # Since there's no filename, this is a subdir -- finish it. oldtree = hashvalid # may be None stack, newtree = stack.pop(override_tree=oldtree) if not oldtree: if lastskip_name and lastskip_name.startswith(ent.name): ent.invalidate() else: ent.validate(GIT_MODE_TREE, newtree) ent.repack() if exists and wasmissing: _nonlocal['count'] += oldsize continue # it's not a directory if hashvalid: meta = msr.metadata_at(ent.meta_ofs) meta.hardlink_target = find_hardlink_target(hlink_db, ent) # Restore the times that were cleared to 0 in the metastore. (meta.atime, meta.mtime, meta.ctime) = (ent.atime, ent.mtime, ent.ctime) stack.append(file, ent.mode, ent.gitmode, ent.sha, meta) else: id = None hlink = find_hardlink_target(hlink_db, ent) try: meta = metadata.from_path( ent.name, hardlink_target=hlink, normalized=True, after_stat=after_nondir_metadata_stat) except (OSError, IOError) as e: add_error(e) lastskip_name = ent.name continue if stat.S_IFMT(ent.mode) != stat.S_IFMT(meta.mode): # The mode changed since we indexed the file, this is bad. # This can cause two issues: # 1) We e.g. think the file is a regular file, but now it's # something else (a device, socket, FIFO or symlink, etc.) # and _read_ from it when we shouldn't. # 2) We then record it as valid, but don't update the index # metadata, and on a subsequent save it has 'hashvalid' # but is recorded as the file type from the index, when # the content is something else ... # Avoid all of these consistency issues by just skipping such # things - it really ought to not happen anyway. add_error("%s: mode changed since indexing, skipping." % path_msg(ent.name)) lastskip_name = ent.name continue if stat.S_ISREG(ent.mode): try: # If the file changes while we're reading it, then our reading # may stop at some point, but the stat() above may have gotten # a different size already. Recalculate the meta size so that # the repository records the accurate size in the metadata, even # if the other stat() data might be slightly older than the file # content (which we can't fix, this is inherently racy, but we # can prevent the size mismatch.) meta.size = 0 def write_data(data): meta.size += len(data) return repo.write_data(data) before_saving_regular_file(ent.name) with hashsplit.open_noatime(ent.name) as f: (mode, id) = hashsplit.split_to_blob_or_tree( write_data, repo.write_tree, [f], keep_boundaries=False, progress=progress_report, blobbits=blobbits) except (IOError, OSError) as e: add_error('%s: %s' % (ent.name, e)) lastskip_name = ent.name elif stat.S_ISDIR(ent.mode): assert (0) # handled above elif stat.S_ISLNK(ent.mode): mode, id = (GIT_MODE_SYMLINK, repo.write_symlink(meta.symlink_target)) else: # Everything else should be fully described by its # metadata, so just record an empty blob, so the paths # in the tree and .bupm will match up. (mode, id) = (GIT_MODE_FILE, repo.write_data(b'')) if id: ent.validate(mode, id) ent.repack() stack.append(file, ent.mode, ent.gitmode, id, meta) if exists and wasmissing: _nonlocal['count'] += oldsize _nonlocal['subcount'] = 0 if opt.progress: pct = total and _nonlocal['count'] * 100.0 / total or 100 progress( 'Saving: %.2f%% (%d/%dk, %d/%d files), done. \n' % (pct, _nonlocal['count'] / 1024, total / 1024, fcount, ftotal)) # pop all parts above the root folder while not stack.parent.nothing: stack, _ = stack.pop() # Finish the root directory. # When there's a collision, use empty metadata for the root. root_meta = metadata.Metadata() if root_collision else None stack, tree = stack.pop(override_meta=root_meta) sys.stdout.flush() out = byte_stream(sys.stdout) if opt.tree: out.write(hexlify(tree)) out.write(b'\n') if opt.commit or name: if compat.py_maj > 2: # Strip b prefix from python 3 bytes reprs to preserve previous format msgcmd = b'[%s]' % b', '.join( [repr(argv_bytes(x))[1:].encode('ascii') for x in argv]) else: msgcmd = repr(argv) msg = b'bup save\n\nGenerated by command:\n%s\n' % msgcmd userline = (b'%s <%s@%s>' % (userfullname(), username(), hostname())) commit = repo.write_commit(tree, oldref, userline, date, None, userline, date, None, msg) if opt.commit: out.write(hexlify(commit)) out.write(b'\n') msr.close() if opt.name: repo.update_ref(refname, commit, oldref) repo.close() if saved_errors: log('WARNING: %d errors encountered while saving.\n' % len(saved_errors)) sys.exit(1)
def main(argv): o = options.Options(optspec) opt, flags, extra = o.parse_bytes(argv[1:]) if opt.name: opt.name = argv_bytes(opt.name) if opt.verbose is None: opt.verbose = 0 if not (opt.blobs or opt.tree or opt.commit or opt.name or opt.noop or opt.copy): o.fatal("use one or more of -b, -t, -c, -n, --noop, --copy") if opt.copy and (opt.blobs or opt.tree): o.fatal('--copy is incompatible with -b, -t') if (opt.noop or opt.copy) and (opt.commit or opt.name): o.fatal('--noop and --copy are incompatible with -c, -n') if opt.blobs and (opt.tree or opt.commit or opt.name): o.fatal('-b is incompatible with -t, -c, -n') if extra and opt.git_ids: o.fatal("don't provide filenames when using --git-ids") if opt.verbose >= 2: git.verbose = opt.verbose - 1 opt.bench = 1 fanout = None if opt.fanout: # This used to be in hashsplit, but that's just confusing; # hashsplit now defaults to the real default (16) if 0 (or # None) is passed, but keep the command-line compatible... fanout = parse_num(opt.fanout) or 128 blobbits = None if opt.blobbits: blobbits = parse_num(opt.blobbits) if opt.bwlimit: client.bwlimit = parse_num(opt.bwlimit) if opt.date: date = parse_date_or_fatal(opt.date, o.fatal) else: date = time.time() # Hack around lack of nonlocal vars in python 2 total_bytes = [0] def prog(filenum, nbytes): total_bytes[0] += nbytes if filenum > 0: qprogress('Splitting: file #%d, %d kbytes\r' % (filenum + 1, total_bytes[0] // 1024)) else: qprogress('Splitting: %d kbytes\r' % (total_bytes[0] // 1024)) start_time = time.time() if opt.name and not valid_save_name(opt.name): o.fatal("'%r' is not a valid branch name." % opt.name) refname = opt.name and b'refs/heads/%s' % opt.name or None if opt.noop or opt.copy: repo = oldref = None else: repo = from_opts(opt) oldref = refname and repo.read_ref(refname) or None repobits = repo.config(b'bup.blobbits', opttype='int') or hashsplit.BUP_BLOBBITS if not blobbits: blobbits = repobits else: print("overriding repo blobbits %d from cmdline with %d" % (repobits, blobbits)) input = byte_stream(sys.stdin) if opt.git_ids: # the input is actually a series of git object ids that we should retrieve # and split. # # This is a bit messy, but basically it converts from a series of # repo.cat() iterators into a series of file-type objects. # It would be less ugly if either repo.cat() returned a file-like object # (not very efficient), or split_to_shalist() expected an iterator instead # of a file. class IterToFile: def __init__(self, it): self.it = iter(it) def read(self, size): v = next(self.it, None) return v or b'' def read_ids(): while 1: line = input.readline() if not line: break if line: line = line.strip() try: it = repo.cat(line.strip()) next(it, None) # skip the file info except KeyError as e: add_error('error: %s' % e) continue yield IterToFile(it) files = read_ids() else: # the input either comes from a series of files or from stdin. files = extra and (open(argv_bytes(fn), 'rb') for fn in extra) or [input] if repo: write_data = repo.write_data write_tree = repo.write_tree elif opt.blobs or opt.tree: # --noop mode write_data = lambda content: git.calc_hash(b'blob', content) write_tree = lambda shalist: git.calc_hash(b'tree', git.tree_encode(shalist)) sys.stdout.flush() out = byte_stream(sys.stdout) if opt.blobs: shalist = hashsplit.split_to_blobs(write_data, files, keep_boundaries=opt.keep_boundaries, progress=prog, blobbits=blobbits) for (sha, size, level) in shalist: out.write(hexlify(sha) + b'\n') reprogress() elif opt.tree or opt.commit or opt.name: if opt.name: # insert dummy_name which may be used as a restore target mode, sha = \ hashsplit.split_to_blob_or_tree(write_data, write_tree, files, keep_boundaries=opt.keep_boundaries, progress=prog, fanout=fanout, blobbits=blobbits) splitfile_name = git.mangle_name(b'data', hashsplit.GIT_MODE_FILE, mode) shalist = [(mode, splitfile_name, sha)] else: shalist = hashsplit.split_to_shalist( write_data, write_tree, files, keep_boundaries=opt.keep_boundaries, progress=prog, fanout=fanout, blobbits=blobbits) tree = write_tree(shalist) else: last = 0 it = hashsplit.hashsplit_iter(files, keep_boundaries=opt.keep_boundaries, progress=prog, fanout=fanout, blobbits=blobbits) for (blob, level) in it: hashsplit.total_split += len(blob) if opt.copy: sys.stdout.write(str(blob)) megs = hashsplit.total_split // 1024 // 1024 if not opt.quiet and last != megs: last = megs if opt.verbose: log('\n') if opt.tree: out.write(hexlify(tree) + b'\n') if opt.commit or opt.name: msg = b'bup split\n\nGenerated by command:\n%r\n' % compat.get_argvb() ref = opt.name and (b'refs/heads/%s' % opt.name) or None userline = b'%s <%s@%s>' % (userfullname(), username(), hostname()) commit = repo.write_commit(tree, oldref, userline, date, None, userline, date, None, msg) if opt.commit: out.write(hexlify(commit) + b'\n') if opt.name and repo: repo.update_ref(refname, commit, oldref) if repo: repo.close() secs = time.time() - start_time size = hashsplit.total_split if opt.bench: log('bup: %.2f kbytes in %.2f secs = %.2f kbytes/sec\n' % (size / 1024, secs, size / 1024 / secs)) if saved_errors: log('WARNING: %d errors encountered while saving.\n' % len(saved_errors)) sys.exit(1)
-- r,remote= remote repository path o= output filename """ o = options.Options(optspec) (opt, flags, extra) = o.parse(sys.argv[1:]) if opt.remote: opt.remote = argv_bytes(opt.remote) stdin = byte_stream(sys.stdin) if not extra: extra = linereader(stdin) ret = 0 repo = from_opts(opt) if opt.o: outfile = open(opt.o, 'wb') else: sys.stdout.flush() outfile = byte_stream(sys.stdout) for ref in [argv_bytes(x) for x in extra]: try: for blob in repo.join(ref): outfile.write(blob) except KeyError as e: outfile.flush() log('error: %s\n' % e) ret = 1
def main(argv): global opt signal.signal(signal.SIGTERM, handle_sigterm) UnixAddress = namedtuple('UnixAddress', ['path']) InetAddress = namedtuple('InetAddress', ['host', 'port']) o = options.Options(optspec) opt, flags, extra = o.parse_bytes(argv[1:]) if len(extra) > 1: o.fatal("at most one argument expected") if len(extra) == 0: address = InetAddress(host='127.0.0.1', port=8080) else: bind_url = extra[0] if bind_url.startswith('unix://'): address = UnixAddress(path=bind_url[len('unix://'):]) else: addr_parts = extra[0].split(':', 1) if len(addr_parts) == 1: host = '127.0.0.1' port = addr_parts[0] else: host, port = addr_parts try: port = int(port) except (TypeError, ValueError) as ex: o.fatal('port must be an integer, not %r' % port) address = InetAddress(host=host, port=port) settings = dict( debug=1, template_path=resource_path(b'web').decode('utf-8'), static_path=resource_path(b'web/static').decode('utf-8'), ) # Disable buffering on stdout, for debug messages try: sys.stdout._line_buffering = True except AttributeError: sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) repo = from_opts(opt, reverse=False) application = tornado.web.Application([ (r"(?P<path>/.*)", BupRequestHandler, dict(repo=repo)), ], **settings) http_server = HTTPServer(application) io_loop_pending = IOLoop.instance() if isinstance(address, InetAddress): sockets = tornado.netutil.bind_sockets(address.port, address.host) http_server.add_sockets(sockets) print('Serving HTTP on %s:%d...' % sockets[0].getsockname()[0:2]) if opt.browser: browser_addr = 'http://' + address[0] + ':' + str(address[1]) io_loop_pending.add_callback(lambda: webbrowser.open(browser_addr)) elif isinstance(address, UnixAddress): unix_socket = bind_unix_socket(address.path) http_server.add_socket(unix_socket) print('Serving HTTP on filesystem socket %r' % address.path) else: log('error: unexpected address %r', address) sys.exit(1) io_loop = io_loop_pending io_loop.start() if saved_errors: log('WARNING: %d errors encountered while saving.\n' % len(saved_errors)) sys.exit(1)
def main(): handle_ctrl_c() opt = parse_args(sys.argv) if opt.source: opt.source = argv_bytes(opt.source) src_dir = opt.source or git.repo() if opt.bwlimit: client.bwlimit = parse_num(opt.bwlimit) if opt.remote: opt.remote = argv_bytes(opt.remote) dest_repo = repo.from_opts(opt) with dest_repo as dest_repo: with LocalRepo(repo_dir=src_dir) as src_repo: # Resolve and validate all sources and destinations, # implicit or explicit, and do it up-front, so we can # fail before we start writing (for any obviously # broken cases). target_items = resolve_targets(opt.target_specs, src_repo, dest_repo) updated_refs = {} # ref_name -> (original_ref, tip_commit(bin)) no_ref_info = (None, None) handlers = { 'ff': handle_ff, 'append': handle_append, 'force-pick': handle_pick, 'pick': handle_pick, 'new-tag': handle_new_tag, 'replace': handle_replace, 'unnamed': handle_unnamed } for item in target_items: debug1('get-spec: %r\n' % (item.spec, )) debug1('get-src: %s\n' % loc_desc(item.src)) debug1('get-dest: %s\n' % loc_desc(item.dest)) dest_path = item.dest and item.dest.path if dest_path: if dest_path.startswith(b'/.tag/'): dest_ref = b'refs/tags/%s' % dest_path[6:] else: dest_ref = b'refs/heads/%s' % dest_path[1:] else: dest_ref = None dest_hash = item.dest and item.dest.hash orig_ref, cur_ref = updated_refs.get(dest_ref, no_ref_info) orig_ref = orig_ref or dest_hash cur_ref = cur_ref or dest_hash handler = handlers[item.spec.method] item_result = handler(item, src_repo, dest_repo, opt) if len(item_result) > 1: new_id, tree = item_result else: new_id = item_result[0] if not dest_ref: log_item(item.spec.src, item.src.type, opt) else: updated_refs[dest_ref] = (orig_ref, new_id) if dest_ref.startswith(b'refs/tags/'): log_item(item.spec.src, item.src.type, opt, tag=new_id) else: log_item(item.spec.src, item.src.type, opt, tree=tree, commit=new_id) # Only update the refs at the very end, once the destination repo # finished writing, so that if something goes wrong above, the old # refs will be undisturbed. for ref_name, info in items(updated_refs): orig_ref, new_ref = info try: dest_repo.update_ref(ref_name, new_ref, orig_ref) if opt.verbose: new_hex = hexlify(new_ref) if orig_ref: orig_hex = hexlify(orig_ref) log('updated %r (%s -> %s)\n' % (ref_name, orig_hex, new_hex)) else: log('updated %r (%s)\n' % (ref_name, new_hex)) except (git.GitError, client.ClientError) as ex: add_error('unable to update ref %r: %s' % (ref_name, ex)) if saved_errors: log('WARNING: %d errors encountered while saving.\n' % len(saved_errors)) sys.exit(1)
def main(argv): o = options.Options(optspec) opt, flags, extra = o.parse_bytes(argv[1:]) r = from_opts(opt) refs = { r[0]: r[1] for r in r.refs() if r[0].startswith(b'refs/tags/') } if opt.delete: # git.delete_ref() doesn't complain if a ref doesn't exist. We # could implement this verification but we'd need to read in the # contents of the tag file and pass the hash, and we already know # about the tag's existance via "tags". tag_name = argv_bytes(opt.delete) refname = b'refs/tags/' + tag_name if not opt.force and refname not in refs: log("error: tag '%s' doesn't exist\n" % path_msg(tag_name)) sys.exit(1) r.delete_ref(b'refs/tags/%s' % tag_name) sys.exit(0) if not extra: for t in refs: sys.stdout.flush() out = byte_stream(sys.stdout) out.write(t[len(b'refs/tags/'):]) out.write(b'\n') sys.exit(0) elif len(extra) != 2: o.fatal('expected commit ref and hash') tag_name, commit = map(argv_bytes, extra[:2]) if not tag_name: o.fatal("tag name must not be empty.") debug1("args: tag name = %s; commit = %s\n" % (path_msg(tag_name), commit.decode('ascii'))) refname = b'refs/tags/' + tag_name if refname in refs and not opt.force: log("bup: error: tag '%s' already exists\n" % path_msg(tag_name)) sys.exit(1) if tag_name.startswith(b'.'): o.fatal("'%s' is not a valid tag name." % path_msg(tag_name)) try: hash = r.rev_parse(commit) except git.GitError as e: log("bup: error: %s" % e) sys.exit(2) if not r.exists(hash): log("bup: error: commit %s not found.\n" % commit.decode('ascii')) sys.exit(2) try: oldval = refs.get(refname, None) r.update_ref(refname, hash, oldval) except git.GitError as e: log("bup: error: could not create tag '%s': %s" % (path_msg(tag_name), e)) sys.exit(3)