def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
stdin = byte_stream(sys.stdin)
if not extra:
extra = linereader(stdin)
ret = 0
repo = from_opts(opt)
if opt.o:
outfile = open(opt.o, 'wb')
else:
sys.stdout.flush()
outfile = byte_stream(sys.stdout)
for ref in [argv_bytes(x) for x in extra]:
try:
for blob in repo.join(ref):
outfile.write(blob)
except KeyError as e:
outfile.flush()
log('error: %s\n' % e)
ret = 1
sys.exit(ret)
def via_cmdline(args, out=None, onabort=None):
"""Write a listing of a file or directory in the bup repository to out.
When a long listing is not requested and stdout is attached to a
tty, the output is formatted in columns. When not attached to tty
(for example when the output is piped to another command), one
file is listed per line.
"""
assert out
opt = opts_from_cmdline(args, onabort=onabort)
r = repo.from_opts(opt, reverse=False)
return within_repo(r, opt, out)
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if not extra:
o.fatal('must specify a target')
if len(extra) > 1:
o.fatal('only one target file allowed')
if opt.bupm and opt.meta:
o.fatal('--meta and --bupm are incompatible')
target = argv_bytes(extra[0])
if not re.match(br'/*[^/]+/[^/]+', target):
o.fatal("path %r doesn't include a branch and revision" % target)
repo = from_opts(opt, reverse=False)
resolved = vfs.resolve(repo, target, follow=False)
leaf_name, leaf_item = resolved[-1]
if not leaf_item:
log('error: cannot access %r in %r\n' %
(b'/'.join(name for name, item in resolved), target))
sys.exit(1)
mode = vfs.item_mode(leaf_item)
sys.stdout.flush()
out = byte_stream(sys.stdout)
if opt.bupm:
if not stat.S_ISDIR(mode):
o.fatal('%r is not a directory' % target)
_, bupm_oid = vfs.tree_data_and_bupm(repo, leaf_item.oid)
if bupm_oid:
with vfs.tree_data_reader(repo, bupm_oid) as meta_stream:
out.write(meta_stream.read())
elif opt.meta:
augmented = vfs.augment_item_meta(repo, leaf_item, include_size=True)
out.write(augmented.meta.encode())
else:
if stat.S_ISREG(mode):
with vfs.fopen(repo, leaf_item) as f:
for b in chunkyreader(f):
out.write(b)
else:
o.fatal('%r is not a plain file' % target)
if saved_errors:
log('warning: %d errors encountered\n' % len(saved_errors))
sys.exit(1)
def main(argv):
o = Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if not opt.unsafe:
o.fatal(
'refusing to run dangerous, experimental command without --unsafe')
if len(extra) < 1:
o.fatal('no paths specified')
repo = from_opts(opt)
bup_rm(repo, [argv_bytes(x) for x in extra], verbosity=opt.verbose)
die_if_errors()
def main(argv):
o = options.Options(optspec)
(opt, flags, extra) = o.parse(argv[1:])
if len(extra) != 1:
o.fatal("must give exactly one name")
name = argv_bytes(extra[0])
r = repo.from_opts(opt)
if opt.type == 'str':
opt.type = None
print("%s = %r" % (name.decode('utf-8'), r.config(name, opttype=opt.type)))
r.close()
parameter = argv_bytes(parameter)
splitted_parameter = parameter.split(b'=')
if len(splitted_parameter) != 2:
o.fatal("a graft point must be of the form old_path=new_path")
old_path, new_path = splitted_parameter
if not (old_path and new_path):
o.fatal("a graft point cannot be empty")
graft_points.append(
(resolve_parent(old_path), resolve_parent(new_path)))
name = opt.name
if name and not valid_save_name(name):
o.fatal("'%s' is not a valid branch name" % path_msg(name))
refname = name and b'refs/heads/%s' % name or None
repo = repo.from_opts(opt)
use_treesplit = repo.config(b'bup.treesplit', opttype='bool')
blobbits = repo.config(b'bup.blobbits', opttype='int')
oldref = refname and repo.read_ref(refname) or None
handle_ctrl_c()
# Metadata is stored in a file named .bupm in each directory. The
# first metadata entry will be the metadata for the current directory.
# The remaining entries will be for each of the other directory
# elements, in the order they're listed in the index.
#
# Since the git tree elements are sorted according to
# git.shalist_item_sort_key, the metalist items are accumulated as
# (sort_key, metadata) tuples, and then sorted when the .bupm file is
def main(argv):
o = options.Options(optspec)
opt, flags, roots = o.parse_bytes(argv[1:])
roots = [argv_bytes(x) for x in roots]
if not opt.unsafe:
o.fatal(
'refusing to run dangerous, experimental command without --unsafe')
now = int(time()) if opt.wrt is None else opt.wrt
if not isinstance(now, int_types):
o.fatal('--wrt value ' + str(now) + ' is not an integer')
period_start = {}
for period, extent in (('all', opt.keep_all_for), ('dailies',
opt.keep_dailies_for),
('monthlies', opt.keep_monthlies_for),
('yearlies', opt.keep_yearlies_for)):
if extent:
secs = period_as_secs(extent.encode('ascii'))
if not secs:
o.fatal('%r is not a valid period' % extent)
period_start[period] = now - secs
if not period_start:
o.fatal('at least one keep argument is required')
period_start = defaultdict(lambda: float('inf'), period_start)
if opt.verbose:
epoch_ymd = strftime('%Y-%m-%d-%H%M%S', localtime(0))
for kind in ['all', 'dailies', 'monthlies', 'yearlies']:
period_utc = period_start[kind]
if period_utc != float('inf'):
if not (period_utc > float('-inf')):
log('keeping all ' + kind)
else:
try:
when = strftime('%Y-%m-%d-%H%M%S',
localtime(period_utc))
log('keeping ' + kind + ' since ' + when + '\n')
except ValueError as ex:
if period_utc < 0:
log('keeping %s since %d seconds before %s\n' %
(kind, abs(period_utc), epoch_ymd))
elif period_utc > 0:
log('keeping %s since %d seconds after %s\n' %
(kind, period_utc, epoch_ymd))
else:
log('keeping %s since %s\n' % (kind, epoch_ymd))
# This could be more efficient, but for now just build the whole list
# in memory and let bup_rm() do some redundant work.
def parse_info(f):
author_secs = f.readline().strip()
return int(author_secs)
sys.stdout.flush()
out = byte_stream(sys.stdout)
repo = from_opts(opt)
removals = []
for branch, branch_id in branches(repo, roots):
die_if_errors()
saves = ((utc, unhexlify(oidx)) for (
oidx,
utc) in repo.rev_list(branch_id, format=b'%at', parse=parse_info))
for keep_save, (utc, id) in classify_saves(saves, period_start):
assert (keep_save in (False, True))
# FIXME: base removals on hashes
if opt.pretend:
out.write((b'+ ' if keep_save else b'- ') +
save_name(branch, utc) + b'\n')
elif not keep_save:
removals.append(save_name(branch, utc))
if not opt.pretend:
die_if_errors()
bup_rm(repo, removals, verbosity=opt.verbose)
if opt.gc:
die_if_errors()
bup_gc(repo,
threshold=opt.gc_threshold,
compression=opt.compress,
verbosity=opt.verbose)
die_if_errors()
def main():
o = options.Options(optspec)
opt, flags, extra = o.parse(sys.argv[1:])
verbosity = (opt.verbose or 0) if not opt.quiet else -1
if opt.remote:
opt.remote = argv_bytes(opt.remote)
if opt.outdir:
opt.outdir = argv_bytes(opt.outdir)
git.check_repo_or_die()
if not extra:
o.fatal('must specify at least one filename to restore')
exclude_rxs = parse_rx_excludes(flags, o.fatal)
owner_map = {}
for map_type in ('user', 'group', 'uid', 'gid'):
owner_map[map_type] = parse_owner_mappings(map_type, flags, o.fatal)
if opt.outdir:
mkdirp(opt.outdir)
os.chdir(opt.outdir)
if opt.remote:
opt.remote = argv_bytes(opt.remote)
repo = from_opts(opt, reverse=False)
top = fsencode(os.getcwd())
hardlinks = {}
for path in [argv_bytes(x) for x in extra]:
if not valid_restore_path(path):
add_error("path %r doesn't include a branch and revision" % path)
continue
try:
resolved = vfs.resolve(repo, path, want_meta=True, follow=False)
except vfs.IOError as e:
add_error(e)
continue
if len(resolved) == 3 and resolved[2][0] == b'latest':
# Follow latest symlink to the actual save
try:
resolved = vfs.resolve(repo,
b'latest',
parent=resolved[:-1],
want_meta=True)
except vfs.IOError as e:
add_error(e)
continue
# Rename it back to 'latest'
resolved = tuple(elt if i != 2 else (b'latest', ) + elt[1:]
for i, elt in enumerate(resolved))
path_parent, path_name = os.path.split(path)
leaf_name, leaf_item = resolved[-1]
if not leaf_item:
add_error('error: cannot access %r in %r' %
('/'.join(name for name, item in resolved), path))
continue
if not path_name or path_name == b'.':
# Source is /foo/what/ever/ or /foo/what/ever/. -- extract
# what/ever/* to the current directory, and if name == '.'
# (i.e. /foo/what/ever/.), then also restore what/ever's
# metadata to the current directory.
treeish = vfs.item_mode(leaf_item)
if not treeish:
add_error('%r cannot be restored as a directory' % path)
else:
items = vfs.contents(repo, leaf_item, want_meta=True)
dot, leaf_item = next(items, None)
assert dot == b'.'
for sub_name, sub_item in items:
restore(repo, b'', sub_name, sub_item, top, opt.sparse,
opt.numeric_ids, owner_map, exclude_rxs, verbosity,
hardlinks)
if path_name == b'.':
leaf_item = vfs.augment_item_meta(repo,
leaf_item,
include_size=True)
apply_metadata(leaf_item.meta, b'.', opt.numeric_ids,
owner_map)
else:
restore(repo, b'', leaf_name, leaf_item, top, opt.sparse,
opt.numeric_ids, owner_map, exclude_rxs, verbosity,
hardlinks)
if verbosity >= 0:
progress('Restoring: %d, done.\n' % total_restored)
die_if_errors()
settings = dict(
debug=1,
template_path=resource_path(b'web').decode('utf-8'),
static_path=resource_path(b'web/static').decode('utf-8'),
)
# Disable buffering on stdout, for debug messages
try:
sys.stdout._line_buffering = True
except AttributeError:
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
if opt.remote:
opt.remote = argv_bytes(opt.remote)
repo = from_opts(opt, reverse=False)
application = tornado.web.Application([
(r"(?P<path>/.*)", BupRequestHandler, dict(repo=repo)),
], **settings)
http_server = HTTPServer(application)
io_loop_pending = IOLoop.instance()
if isinstance(address, InetAddress):
sockets = tornado.netutil.bind_sockets(address.port, address.host)
http_server.add_sockets(sockets)
print('Serving HTTP on %s:%d...' % sockets[0].getsockname())
if opt.browser:
browser_addr = 'http://' + address[0] + ':' + str(address[1])
io_loop_pending.add_callback(lambda: webbrowser.open(browser_addr))
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
global repo
repo = from_opts(opt, reverse=False)
sys.stdout.flush()
out = byte_stream(sys.stdout)
stdin = byte_stream(sys.stdin)
pwd = vfs.resolve(repo, b'/')
rv = 0
def inputiter(f):
if os.isatty(f.fileno()):
while 1:
prompt = b'bup %s> ' % (b'/'.join(name for name, item in pwd) or b'/', )
if hasattr(_helpers, 'readline'):
try:
yield _helpers.readline(prompt)
except EOFError:
print() # Clear the line for the terminal's next prompt
break
else:
out.write(prompt)
out.flush()
read_line = f.readline()
if not read_line:
print('')
break
yield read_line
else:
for line in f:
yield line
if extra:
lines = (argv_bytes(arg) for arg in extra)
else:
if hasattr(_helpers, 'readline'):
_helpers.set_completer_word_break_characters(b' \t\n\r/')
_helpers.set_attempted_completion_function(attempt_completion)
_helpers.set_completion_entry_function(enter_completion)
if sys.platform.startswith('darwin'):
# MacOS uses a slightly incompatible clone of libreadline
_helpers.parse_and_bind(b'bind ^I rl_complete')
_helpers.parse_and_bind(b'tab: complete')
lines = inputiter(stdin)
for line in lines:
if not line.strip():
continue
words = [word for (wordstart,word) in shquote.quotesplit(line)]
cmd = words[0].lower()
#log('execute: %r %r\n' % (cmd, parm))
try:
if cmd == b'ls':
do_ls(repo, pwd, words[1:], out)
out.flush()
elif cmd == b'cd':
np = pwd
for parm in words[1:]:
res = vfs.resolve(repo, parm, parent=np)
_, leaf_item = res[-1]
if not leaf_item:
raise CommandError(b'"%s" does not exist' %
b'/'.join(name for name, item in res))
if not stat.S_ISDIR(vfs.item_mode(leaf_item)):
raise CommandError(b'"%s" is not a directory' % parm)
np = res
pwd = np
elif cmd == b'pwd':
if len(pwd) == 1:
out.write(b'/')
out.write(b'/'.join(name for name, item in pwd) + b'\n')
out.flush()
elif cmd == b'cat':
for parm in words[1:]:
res = vfs.resolve(repo, parm, parent=pwd)
_, leaf_item = res[-1]
if not leaf_item:
raise CommandError(b'"%s" does not exist' %
b'/'.join(name for name, item in res))
with vfs.fopen(repo, leaf_item) as srcfile:
write_to_file(srcfile, out)
out.flush()
elif cmd == b'get':
if len(words) not in [2,3]:
rv = 1
raise CommandError(b'Usage: get <filename> [localname]')
rname = words[1]
(dir,base) = os.path.split(rname)
lname = len(words) > 2 and words[2] or base
res = vfs.resolve(repo, rname, parent=pwd)
_, leaf_item = res[-1]
if not leaf_item:
raise CommandError(b'"%s" does not exist' %
b'/'.join(name for name, item in res))
with vfs.fopen(repo, leaf_item) as srcfile:
with open(lname, 'wb') as destfile:
log('Saving %s\n' % path_msg(lname))
write_to_file(srcfile, destfile)
elif cmd == b'mget':
for parm in words[1:]:
dir, base = os.path.split(parm)
res = vfs.resolve(repo, dir, parent=pwd)
_, dir_item = res[-1]
if not dir_item:
raise CommandError(b'"%s" does not exist' % dir)
for name, item in vfs.contents(repo, dir_item):
if name == b'.':
continue
if fnmatch.fnmatch(name, base):
if stat.S_ISLNK(vfs.item_mode(item)):
deref = vfs.resolve(repo, name, parent=res)
deref_name, deref_item = deref[-1]
if not deref_item:
raise CommandError(b'"%s" does not exist' %
b'/'.join(name for name, item in res))
item = deref_item
with vfs.fopen(repo, item) as srcfile:
with open(name, 'wb') as destfile:
log('Saving %s\n' % path_msg(name))
write_to_file(srcfile, destfile)
elif cmd == b'help' or cmd == b'?':
out.write(b'Commands: ls cd pwd cat get mget help quit\n')
out.flush()
elif cmd in (b'quit', b'exit', b'bye'):
break
else:
rv = 1
raise CommandError(b'no such command "%s"' % cmd)
except CommandError as e:
rv = 1
out.write(b'error: %s\n' % e.args[0])
out.flush()
except Exception as e:
rv = 1
out.write(b'error: %s\n' % str(e).encode())
out.flush()
sys.exit(rv)
def main(argv):
# Hack around lack of nonlocal vars in python 2
_nonlocal = {}
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if opt.indexfile:
opt.indexfile = argv_bytes(opt.indexfile)
if opt.name:
opt.name = argv_bytes(opt.name)
if opt.strip_path:
opt.strip_path = argv_bytes(opt.strip_path)
if not (opt.tree or opt.commit or opt.name):
o.fatal("use one or more of -t, -c, -n")
if not extra:
o.fatal("no filenames given")
extra = [argv_bytes(x) for x in extra]
opt.progress = (istty2 and not opt.quiet)
opt.smaller = parse_num(opt.smaller or 0)
if opt.bwlimit:
client.bwlimit = parse_num(opt.bwlimit)
if opt.date:
date = parse_date_or_fatal(opt.date, o.fatal)
else:
date = time.time()
if opt.strip and opt.strip_path:
o.fatal("--strip is incompatible with --strip-path")
graft_points = []
if opt.graft:
if opt.strip:
o.fatal("--strip is incompatible with --graft")
if opt.strip_path:
o.fatal("--strip-path is incompatible with --graft")
for (option, parameter) in flags:
if option == "--graft":
parameter = argv_bytes(parameter)
splitted_parameter = parameter.split(b'=')
if len(splitted_parameter) != 2:
o.fatal(
"a graft point must be of the form old_path=new_path")
old_path, new_path = splitted_parameter
if not (old_path and new_path):
o.fatal("a graft point cannot be empty")
graft_points.append(
(resolve_parent(old_path), resolve_parent(new_path)))
name = opt.name
if name and not valid_save_name(name):
o.fatal("'%s' is not a valid branch name" % path_msg(name))
refname = name and b'refs/heads/%s' % name or None
repo = from_opts(opt)
blobbits = repo.config(b'bup.blobbits', opttype='int')
oldref = refname and repo.read_ref(refname) or None
handle_ctrl_c()
# Metadata is stored in a file named .bupm in each directory. The
# first metadata entry will be the metadata for the current directory.
# The remaining entries will be for each of the other directory
# elements, in the order they're listed in the index.
#
# Since the git tree elements are sorted according to
# git.shalist_item_sort_key, the metalist items are accumulated as
# (sort_key, metadata) tuples, and then sorted when the .bupm file is
# created. The sort_key should have been computed using the element's
# mangled name and git mode (after hashsplitting), but the code isn't
# actually doing that but rather uses the element's real name and mode.
# This makes things a bit more difficult when reading it back, see
# vfs.ordered_tree_entries().
# Maintain a stack of information representing the current location in
# the archive being constructed.
stack = Stack(repo)
_nonlocal['count'] = 0
_nonlocal['subcount'] = 0
_nonlocal['lastremain'] = None
def progress_report(file, n):
_nonlocal['subcount'] += n
cc = _nonlocal['count'] + _nonlocal['subcount']
pct = total and (cc * 100.0 / total) or 0
now = time.time()
elapsed = now - tstart
kps = elapsed and int(cc / 1024. / elapsed)
kps_frac = 10**int(math.log(kps + 1, 10) - 1)
kps = int(kps / kps_frac) * kps_frac
if cc:
remain = elapsed * 1.0 / cc * (total - cc)
else:
remain = 0.0
if (_nonlocal['lastremain'] and (remain > _nonlocal['lastremain']) and
((remain - _nonlocal['lastremain']) / _nonlocal['lastremain'] <
0.05)):
remain = _nonlocal['lastremain']
else:
_nonlocal['lastremain'] = remain
hours = int(remain / 60 / 60)
mins = int(remain / 60 - hours * 60)
secs = int(remain - hours * 60 * 60 - mins * 60)
if elapsed < 30:
remainstr = ''
kpsstr = ''
else:
kpsstr = '%dk/s' % kps
if hours:
remainstr = '%dh%dm' % (hours, mins)
elif mins:
remainstr = '%dm%d' % (mins, secs)
else:
remainstr = '%ds' % secs
qprogress(
'Saving: %.2f%% (%d/%dk, %d/%d files) %s %s\r' %
(pct, cc / 1024, total / 1024, fcount, ftotal, remainstr, kpsstr))
indexfile = opt.indexfile or path.index()
r = index.Reader(indexfile)
try:
msr = index.MetaStoreReader(indexfile + b'.meta')
except IOError as ex:
if ex.errno != EACCES:
raise
log('error: cannot access %r; have you run bup index?' %
path_msg(indexfile))
sys.exit(1)
hlink_db = hlinkdb.HLinkDB(indexfile + b'.hlink')
def already_saved(ent):
return ent.is_valid() and repo.exists(ent.sha) and ent.sha
def wantrecurse_pre(ent):
return not already_saved(ent)
def wantrecurse_during(ent):
return not already_saved(ent) or ent.sha_missing()
def find_hardlink_target(hlink_db, ent):
if hlink_db and not stat.S_ISDIR(ent.mode) and ent.nlink > 1:
link_paths = hlink_db.node_paths(ent.dev, ent.ino)
if link_paths:
return link_paths[0]
total = ftotal = 0
if opt.progress:
for (transname, ent) in r.filter(extra, wantrecurse=wantrecurse_pre):
if not (ftotal % 10024):
qprogress('Reading index: %d\r' % ftotal)
exists = ent.exists()
hashvalid = already_saved(ent)
ent.set_sha_missing(not hashvalid)
if not opt.smaller or ent.size < opt.smaller:
if exists and not hashvalid:
total += ent.size
ftotal += 1
progress('Reading index: %d, done.\n' % ftotal)
# Root collisions occur when strip or graft options map more than one
# path to the same directory (paths which originally had separate
# parents). When that situation is detected, use empty metadata for
# the parent. Otherwise, use the metadata for the common parent.
# Collision example: "bup save ... --strip /foo /foo/bar /bar".
# FIXME: Add collision tests, or handle collisions some other way.
# FIXME: Detect/handle strip/graft name collisions (other than root),
# i.e. if '/foo/bar' and '/bar' both map to '/'.
first_root = None
root_collision = None
tstart = time.time()
fcount = 0
lastskip_name = None
lastdir = b''
for (transname, ent) in r.filter(extra, wantrecurse=wantrecurse_during):
(dir, file) = os.path.split(ent.name)
exists = (ent.flags & index.IX_EXISTS)
hashvalid = already_saved(ent)
wasmissing = ent.sha_missing()
oldsize = ent.size
if opt.verbose:
if not exists:
status = 'D'
elif not hashvalid:
if ent.sha == index.EMPTY_SHA:
status = 'A'
else:
status = 'M'
else:
status = ' '
if opt.verbose >= 2:
log('%s %-70s\n' % (status, path_msg(ent.name)))
elif not stat.S_ISDIR(ent.mode) and lastdir != dir:
if not lastdir.startswith(dir):
log('%s %-70s\n' %
(status, path_msg(os.path.join(dir, b''))))
lastdir = dir
if not opt.progress:
progress_report = None
fcount += 1
if not exists:
continue
if opt.smaller and ent.size >= opt.smaller:
if exists and not hashvalid:
if opt.verbose:
log('skipping large file "%s"\n' % path_msg(ent.name))
lastskip_name = ent.name
continue
assert (dir.startswith(b'/'))
if opt.strip:
dirp = stripped_path_components(dir, extra)
elif opt.strip_path:
dirp = stripped_path_components(dir, [opt.strip_path])
elif graft_points:
dirp = grafted_path_components(graft_points, dir)
else:
dirp = path_components(dir)
# At this point, dirp contains a representation of the archive
# path that looks like [(archive_dir_name, real_fs_path), ...].
# So given "bup save ... --strip /foo/bar /foo/bar/baz", dirp
# might look like this at some point:
# [('', '/foo/bar'), ('baz', '/foo/bar/baz'), ...].
# This dual representation supports stripping/grafting, where the
# archive path may not have a direct correspondence with the
# filesystem. The root directory is represented by an initial
# component named '', and any component that doesn't have a
# corresponding filesystem directory (due to grafting, for
# example) will have a real_fs_path of None, i.e. [('', None),
# ...].
if first_root == None:
first_root = dirp[0]
elif first_root != dirp[0]:
root_collision = True
# If switching to a new sub-tree, finish the current sub-tree.
while list(stack.namestack) > [x[0] for x in dirp]:
stack, _ = stack.pop()
# If switching to a new sub-tree, start a new sub-tree.
for path_component in dirp[len(stack):]:
dir_name, fs_path = path_component
# Not indexed, so just grab the FS metadata or use empty metadata.
try:
meta = metadata.from_path(fs_path, normalized=True) \
if fs_path else metadata.Metadata()
except (OSError, IOError) as e:
add_error(e)
lastskip_name = dir_name
meta = metadata.Metadata()
stack = stack.push(dir_name, meta)
if not file:
if len(stack) == 1:
continue # We're at the top level -- keep the current root dir
# Since there's no filename, this is a subdir -- finish it.
oldtree = hashvalid # may be None
stack, newtree = stack.pop(override_tree=oldtree)
if not oldtree:
if lastskip_name and lastskip_name.startswith(ent.name):
ent.invalidate()
else:
ent.validate(GIT_MODE_TREE, newtree)
ent.repack()
if exists and wasmissing:
_nonlocal['count'] += oldsize
continue
# it's not a directory
if hashvalid:
meta = msr.metadata_at(ent.meta_ofs)
meta.hardlink_target = find_hardlink_target(hlink_db, ent)
# Restore the times that were cleared to 0 in the metastore.
(meta.atime, meta.mtime, meta.ctime) = (ent.atime, ent.mtime,
ent.ctime)
stack.append(file, ent.mode, ent.gitmode, ent.sha, meta)
else:
id = None
hlink = find_hardlink_target(hlink_db, ent)
try:
meta = metadata.from_path(
ent.name,
hardlink_target=hlink,
normalized=True,
after_stat=after_nondir_metadata_stat)
except (OSError, IOError) as e:
add_error(e)
lastskip_name = ent.name
continue
if stat.S_IFMT(ent.mode) != stat.S_IFMT(meta.mode):
# The mode changed since we indexed the file, this is bad.
# This can cause two issues:
# 1) We e.g. think the file is a regular file, but now it's
# something else (a device, socket, FIFO or symlink, etc.)
# and _read_ from it when we shouldn't.
# 2) We then record it as valid, but don't update the index
# metadata, and on a subsequent save it has 'hashvalid'
# but is recorded as the file type from the index, when
# the content is something else ...
# Avoid all of these consistency issues by just skipping such
# things - it really ought to not happen anyway.
add_error("%s: mode changed since indexing, skipping." %
path_msg(ent.name))
lastskip_name = ent.name
continue
if stat.S_ISREG(ent.mode):
try:
# If the file changes while we're reading it, then our reading
# may stop at some point, but the stat() above may have gotten
# a different size already. Recalculate the meta size so that
# the repository records the accurate size in the metadata, even
# if the other stat() data might be slightly older than the file
# content (which we can't fix, this is inherently racy, but we
# can prevent the size mismatch.)
meta.size = 0
def write_data(data):
meta.size += len(data)
return repo.write_data(data)
before_saving_regular_file(ent.name)
with hashsplit.open_noatime(ent.name) as f:
(mode, id) = hashsplit.split_to_blob_or_tree(
write_data,
repo.write_tree, [f],
keep_boundaries=False,
progress=progress_report,
blobbits=blobbits)
except (IOError, OSError) as e:
add_error('%s: %s' % (ent.name, e))
lastskip_name = ent.name
elif stat.S_ISDIR(ent.mode):
assert (0) # handled above
elif stat.S_ISLNK(ent.mode):
mode, id = (GIT_MODE_SYMLINK,
repo.write_symlink(meta.symlink_target))
else:
# Everything else should be fully described by its
# metadata, so just record an empty blob, so the paths
# in the tree and .bupm will match up.
(mode, id) = (GIT_MODE_FILE, repo.write_data(b''))
if id:
ent.validate(mode, id)
ent.repack()
stack.append(file, ent.mode, ent.gitmode, id, meta)
if exists and wasmissing:
_nonlocal['count'] += oldsize
_nonlocal['subcount'] = 0
if opt.progress:
pct = total and _nonlocal['count'] * 100.0 / total or 100
progress(
'Saving: %.2f%% (%d/%dk, %d/%d files), done. \n' %
(pct, _nonlocal['count'] / 1024, total / 1024, fcount, ftotal))
# pop all parts above the root folder
while not stack.parent.nothing:
stack, _ = stack.pop()
# Finish the root directory.
# When there's a collision, use empty metadata for the root.
root_meta = metadata.Metadata() if root_collision else None
stack, tree = stack.pop(override_meta=root_meta)
sys.stdout.flush()
out = byte_stream(sys.stdout)
if opt.tree:
out.write(hexlify(tree))
out.write(b'\n')
if opt.commit or name:
if compat.py_maj > 2:
# Strip b prefix from python 3 bytes reprs to preserve previous format
msgcmd = b'[%s]' % b', '.join(
[repr(argv_bytes(x))[1:].encode('ascii') for x in argv])
else:
msgcmd = repr(argv)
msg = b'bup save\n\nGenerated by command:\n%s\n' % msgcmd
userline = (b'%s <%s@%s>' % (userfullname(), username(), hostname()))
commit = repo.write_commit(tree, oldref, userline, date, None,
userline, date, None, msg)
if opt.commit:
out.write(hexlify(commit))
out.write(b'\n')
msr.close()
if opt.name:
repo.update_ref(refname, commit, oldref)
repo.close()
if saved_errors:
log('WARNING: %d errors encountered while saving.\n' %
len(saved_errors))
sys.exit(1)
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if opt.name: opt.name = argv_bytes(opt.name)
if opt.verbose is None: opt.verbose = 0
if not (opt.blobs or opt.tree or opt.commit or opt.name or opt.noop
or opt.copy):
o.fatal("use one or more of -b, -t, -c, -n, --noop, --copy")
if opt.copy and (opt.blobs or opt.tree):
o.fatal('--copy is incompatible with -b, -t')
if (opt.noop or opt.copy) and (opt.commit or opt.name):
o.fatal('--noop and --copy are incompatible with -c, -n')
if opt.blobs and (opt.tree or opt.commit or opt.name):
o.fatal('-b is incompatible with -t, -c, -n')
if extra and opt.git_ids:
o.fatal("don't provide filenames when using --git-ids")
if opt.verbose >= 2:
git.verbose = opt.verbose - 1
opt.bench = 1
fanout = None
if opt.fanout:
# This used to be in hashsplit, but that's just confusing;
# hashsplit now defaults to the real default (16) if 0 (or
# None) is passed, but keep the command-line compatible...
fanout = parse_num(opt.fanout) or 128
blobbits = None
if opt.blobbits:
blobbits = parse_num(opt.blobbits)
if opt.bwlimit:
client.bwlimit = parse_num(opt.bwlimit)
if opt.date:
date = parse_date_or_fatal(opt.date, o.fatal)
else:
date = time.time()
# Hack around lack of nonlocal vars in python 2
total_bytes = [0]
def prog(filenum, nbytes):
total_bytes[0] += nbytes
if filenum > 0:
qprogress('Splitting: file #%d, %d kbytes\r' %
(filenum + 1, total_bytes[0] // 1024))
else:
qprogress('Splitting: %d kbytes\r' % (total_bytes[0] // 1024))
start_time = time.time()
if opt.name and not valid_save_name(opt.name):
o.fatal("'%r' is not a valid branch name." % opt.name)
refname = opt.name and b'refs/heads/%s' % opt.name or None
if opt.noop or opt.copy:
repo = oldref = None
else:
repo = from_opts(opt)
oldref = refname and repo.read_ref(refname) or None
repobits = repo.config(b'bup.blobbits',
opttype='int') or hashsplit.BUP_BLOBBITS
if not blobbits:
blobbits = repobits
else:
print("overriding repo blobbits %d from cmdline with %d" %
(repobits, blobbits))
input = byte_stream(sys.stdin)
if opt.git_ids:
# the input is actually a series of git object ids that we should retrieve
# and split.
#
# This is a bit messy, but basically it converts from a series of
# repo.cat() iterators into a series of file-type objects.
# It would be less ugly if either repo.cat() returned a file-like object
# (not very efficient), or split_to_shalist() expected an iterator instead
# of a file.
class IterToFile:
def __init__(self, it):
self.it = iter(it)
def read(self, size):
v = next(self.it, None)
return v or b''
def read_ids():
while 1:
line = input.readline()
if not line:
break
if line:
line = line.strip()
try:
it = repo.cat(line.strip())
next(it, None) # skip the file info
except KeyError as e:
add_error('error: %s' % e)
continue
yield IterToFile(it)
files = read_ids()
else:
# the input either comes from a series of files or from stdin.
files = extra and (open(argv_bytes(fn), 'rb')
for fn in extra) or [input]
if repo:
write_data = repo.write_data
write_tree = repo.write_tree
elif opt.blobs or opt.tree:
# --noop mode
write_data = lambda content: git.calc_hash(b'blob', content)
write_tree = lambda shalist: git.calc_hash(b'tree',
git.tree_encode(shalist))
sys.stdout.flush()
out = byte_stream(sys.stdout)
if opt.blobs:
shalist = hashsplit.split_to_blobs(write_data,
files,
keep_boundaries=opt.keep_boundaries,
progress=prog,
blobbits=blobbits)
for (sha, size, level) in shalist:
out.write(hexlify(sha) + b'\n')
reprogress()
elif opt.tree or opt.commit or opt.name:
if opt.name: # insert dummy_name which may be used as a restore target
mode, sha = \
hashsplit.split_to_blob_or_tree(write_data, write_tree, files,
keep_boundaries=opt.keep_boundaries,
progress=prog, fanout=fanout,
blobbits=blobbits)
splitfile_name = git.mangle_name(b'data', hashsplit.GIT_MODE_FILE,
mode)
shalist = [(mode, splitfile_name, sha)]
else:
shalist = hashsplit.split_to_shalist(
write_data,
write_tree,
files,
keep_boundaries=opt.keep_boundaries,
progress=prog,
fanout=fanout,
blobbits=blobbits)
tree = write_tree(shalist)
else:
last = 0
it = hashsplit.hashsplit_iter(files,
keep_boundaries=opt.keep_boundaries,
progress=prog,
fanout=fanout,
blobbits=blobbits)
for (blob, level) in it:
hashsplit.total_split += len(blob)
if opt.copy:
sys.stdout.write(str(blob))
megs = hashsplit.total_split // 1024 // 1024
if not opt.quiet and last != megs:
last = megs
if opt.verbose:
log('\n')
if opt.tree:
out.write(hexlify(tree) + b'\n')
if opt.commit or opt.name:
msg = b'bup split\n\nGenerated by command:\n%r\n' % compat.get_argvb()
ref = opt.name and (b'refs/heads/%s' % opt.name) or None
userline = b'%s <%s@%s>' % (userfullname(), username(), hostname())
commit = repo.write_commit(tree, oldref, userline, date, None,
userline, date, None, msg)
if opt.commit:
out.write(hexlify(commit) + b'\n')
if opt.name and repo:
repo.update_ref(refname, commit, oldref)
if repo:
repo.close()
secs = time.time() - start_time
size = hashsplit.total_split
if opt.bench:
log('bup: %.2f kbytes in %.2f secs = %.2f kbytes/sec\n' %
(size / 1024, secs, size / 1024 / secs))
if saved_errors:
log('WARNING: %d errors encountered while saving.\n' %
len(saved_errors))
sys.exit(1)
--
r,remote= remote repository path
o= output filename
"""
o = options.Options(optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])
if opt.remote:
opt.remote = argv_bytes(opt.remote)
stdin = byte_stream(sys.stdin)
if not extra:
extra = linereader(stdin)
ret = 0
repo = from_opts(opt)
if opt.o:
outfile = open(opt.o, 'wb')
else:
sys.stdout.flush()
outfile = byte_stream(sys.stdout)
for ref in [argv_bytes(x) for x in extra]:
try:
for blob in repo.join(ref):
outfile.write(blob)
except KeyError as e:
outfile.flush()
log('error: %s\n' % e)
ret = 1
def main(argv):
global opt
signal.signal(signal.SIGTERM, handle_sigterm)
UnixAddress = namedtuple('UnixAddress', ['path'])
InetAddress = namedtuple('InetAddress', ['host', 'port'])
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if len(extra) > 1:
o.fatal("at most one argument expected")
if len(extra) == 0:
address = InetAddress(host='127.0.0.1', port=8080)
else:
bind_url = extra[0]
if bind_url.startswith('unix://'):
address = UnixAddress(path=bind_url[len('unix://'):])
else:
addr_parts = extra[0].split(':', 1)
if len(addr_parts) == 1:
host = '127.0.0.1'
port = addr_parts[0]
else:
host, port = addr_parts
try:
port = int(port)
except (TypeError, ValueError) as ex:
o.fatal('port must be an integer, not %r' % port)
address = InetAddress(host=host, port=port)
settings = dict(
debug=1,
template_path=resource_path(b'web').decode('utf-8'),
static_path=resource_path(b'web/static').decode('utf-8'),
)
# Disable buffering on stdout, for debug messages
try:
sys.stdout._line_buffering = True
except AttributeError:
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
repo = from_opts(opt, reverse=False)
application = tornado.web.Application([
(r"(?P<path>/.*)", BupRequestHandler, dict(repo=repo)),
], **settings)
http_server = HTTPServer(application)
io_loop_pending = IOLoop.instance()
if isinstance(address, InetAddress):
sockets = tornado.netutil.bind_sockets(address.port, address.host)
http_server.add_sockets(sockets)
print('Serving HTTP on %s:%d...' % sockets[0].getsockname()[0:2])
if opt.browser:
browser_addr = 'http://' + address[0] + ':' + str(address[1])
io_loop_pending.add_callback(lambda: webbrowser.open(browser_addr))
elif isinstance(address, UnixAddress):
unix_socket = bind_unix_socket(address.path)
http_server.add_socket(unix_socket)
print('Serving HTTP on filesystem socket %r' % address.path)
else:
log('error: unexpected address %r', address)
sys.exit(1)
io_loop = io_loop_pending
io_loop.start()
if saved_errors:
log('WARNING: %d errors encountered while saving.\n' %
len(saved_errors))
sys.exit(1)
def main():
handle_ctrl_c()
opt = parse_args(sys.argv)
if opt.source:
opt.source = argv_bytes(opt.source)
src_dir = opt.source or git.repo()
if opt.bwlimit:
client.bwlimit = parse_num(opt.bwlimit)
if opt.remote:
opt.remote = argv_bytes(opt.remote)
dest_repo = repo.from_opts(opt)
with dest_repo as dest_repo:
with LocalRepo(repo_dir=src_dir) as src_repo:
# Resolve and validate all sources and destinations,
# implicit or explicit, and do it up-front, so we can
# fail before we start writing (for any obviously
# broken cases).
target_items = resolve_targets(opt.target_specs, src_repo,
dest_repo)
updated_refs = {} # ref_name -> (original_ref, tip_commit(bin))
no_ref_info = (None, None)
handlers = {
'ff': handle_ff,
'append': handle_append,
'force-pick': handle_pick,
'pick': handle_pick,
'new-tag': handle_new_tag,
'replace': handle_replace,
'unnamed': handle_unnamed
}
for item in target_items:
debug1('get-spec: %r\n' % (item.spec, ))
debug1('get-src: %s\n' % loc_desc(item.src))
debug1('get-dest: %s\n' % loc_desc(item.dest))
dest_path = item.dest and item.dest.path
if dest_path:
if dest_path.startswith(b'/.tag/'):
dest_ref = b'refs/tags/%s' % dest_path[6:]
else:
dest_ref = b'refs/heads/%s' % dest_path[1:]
else:
dest_ref = None
dest_hash = item.dest and item.dest.hash
orig_ref, cur_ref = updated_refs.get(dest_ref, no_ref_info)
orig_ref = orig_ref or dest_hash
cur_ref = cur_ref or dest_hash
handler = handlers[item.spec.method]
item_result = handler(item, src_repo, dest_repo, opt)
if len(item_result) > 1:
new_id, tree = item_result
else:
new_id = item_result[0]
if not dest_ref:
log_item(item.spec.src, item.src.type, opt)
else:
updated_refs[dest_ref] = (orig_ref, new_id)
if dest_ref.startswith(b'refs/tags/'):
log_item(item.spec.src, item.src.type, opt, tag=new_id)
else:
log_item(item.spec.src,
item.src.type,
opt,
tree=tree,
commit=new_id)
# Only update the refs at the very end, once the destination repo
# finished writing, so that if something goes wrong above, the old
# refs will be undisturbed.
for ref_name, info in items(updated_refs):
orig_ref, new_ref = info
try:
dest_repo.update_ref(ref_name, new_ref, orig_ref)
if opt.verbose:
new_hex = hexlify(new_ref)
if orig_ref:
orig_hex = hexlify(orig_ref)
log('updated %r (%s -> %s)\n' %
(ref_name, orig_hex, new_hex))
else:
log('updated %r (%s)\n' % (ref_name, new_hex))
except (git.GitError, client.ClientError) as ex:
add_error('unable to update ref %r: %s' % (ref_name, ex))
if saved_errors:
log('WARNING: %d errors encountered while saving.\n' %
len(saved_errors))
sys.exit(1)
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
r = from_opts(opt)
refs = { r[0]: r[1] for r in r.refs() if r[0].startswith(b'refs/tags/') }
if opt.delete:
# git.delete_ref() doesn't complain if a ref doesn't exist. We
# could implement this verification but we'd need to read in the
# contents of the tag file and pass the hash, and we already know
# about the tag's existance via "tags".
tag_name = argv_bytes(opt.delete)
refname = b'refs/tags/' + tag_name
if not opt.force and refname not in refs:
log("error: tag '%s' doesn't exist\n" % path_msg(tag_name))
sys.exit(1)
r.delete_ref(b'refs/tags/%s' % tag_name)
sys.exit(0)
if not extra:
for t in refs:
sys.stdout.flush()
out = byte_stream(sys.stdout)
out.write(t[len(b'refs/tags/'):])
out.write(b'\n')
sys.exit(0)
elif len(extra) != 2:
o.fatal('expected commit ref and hash')
tag_name, commit = map(argv_bytes, extra[:2])
if not tag_name:
o.fatal("tag name must not be empty.")
debug1("args: tag name = %s; commit = %s\n"
% (path_msg(tag_name), commit.decode('ascii')))
refname = b'refs/tags/' + tag_name
if refname in refs and not opt.force:
log("bup: error: tag '%s' already exists\n" % path_msg(tag_name))
sys.exit(1)
if tag_name.startswith(b'.'):
o.fatal("'%s' is not a valid tag name." % path_msg(tag_name))
try:
hash = r.rev_parse(commit)
except git.GitError as e:
log("bup: error: %s" % e)
sys.exit(2)
if not r.exists(hash):
log("bup: error: commit %s not found.\n" % commit.decode('ascii'))
sys.exit(2)
try:
oldval = refs.get(refname, None)
r.update_ref(refname, hash, oldval)
except git.GitError as e:
log("bup: error: could not create tag '%s': %s" % (path_msg(tag_name), e))
sys.exit(3)
