def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if extra:
o.fatal('no arguments expected')
debug2('bup server: reading from stdin.\n')
# FIXME: this protocol is totally lame and not at all future-proof.
# (Especially since we abort completely as soon as *anything* bad happens)
sys.stdout.flush()
conn = Conn(byte_stream(sys.stdin), byte_stream(sys.stdout))
lr = linereader(conn)
for _line in lr:
line = _line.strip()
if not line:
continue
debug1('bup server: command: %r\n' % line)
words = line.split(b' ', 1)
cmd = words[0]
rest = len(words)>1 and words[1] or b''
if cmd == b'quit':
break
else:
cmd = commands.get(cmd)
if cmd:
cmd(conn, rest)
else:
raise Exception('unknown server command: %r\n' % line)
debug1('bup server: done\n')
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if opt.remote:
opt.remote = argv_bytes(opt.remote)
git.check_repo_or_die()
stdin = byte_stream(sys.stdin)
if not extra:
extra = linereader(stdin)
ret = 0
repo = RemoteRepo(opt.remote) if opt.remote else LocalRepo()
if opt.o:
outfile = open(opt.o, 'wb')
else:
sys.stdout.flush()
outfile = byte_stream(sys.stdout)
for ref in [argv_bytes(x) for x in extra]:
try:
for blob in repo.join(ref):
outfile.write(blob)
except KeyError as e:
outfile.flush()
log('error: %s\n' % e)
ret = 1
sys.exit(ret)
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if extra:
o.fatal('no arguments expected')
debug2('bup server: reading from stdin.\n')
class ServerRepo(LocalRepo):
def __init__(self, repo_dir):
if opt.force_repo:
repo_dir = None
LocalRepo.__init__(self, repo_dir)
def _restrict(server, commands):
for fn in dir(server):
if getattr(fn, 'bup_server_command', False):
if not fn in commands:
del cls.fn
modes = ['unrestricted', 'append', 'read-append', 'read']
if opt.mode is not None and opt.mode not in modes:
o.fatal("server: invalid mode")
BupProtocolServer(Conn(byte_stream(sys.stdin), byte_stream(sys.stdout)),
ServerRepo, mode=opt.mode).handle()
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
opt.dir = argv_bytes(opt.dir) if opt.dir else None
opt.output = argv_bytes(opt.output) if opt.output else None
if extra and (opt.auto or opt.force):
o.fatal("you can't use -f/-a and also provide filenames")
if opt.check and (not extra and not opt.auto):
o.fatal("if using --check, you must provide filenames or -a")
git.check_repo_or_die()
if opt.max_files < 0:
opt.max_files = max_files()
assert (opt.max_files >= 5)
extra = [argv_bytes(x) for x in extra]
if opt.check:
# check existing midx files
if extra:
midxes = extra
else:
path = opt.dir or git.repo(b'objects/pack')
debug1('midx: scanning %s\n' % path)
midxes = glob.glob(os.path.join(path, b'*.midx'))
for name in midxes:
check_midx(name)
if not saved_errors:
log('All tests passed.\n')
else:
if extra:
sys.stdout.flush()
do_midx(git.repo(b'objects/pack'),
opt.output,
extra,
b'',
byte_stream(sys.stdout),
auto=opt.auto,
force=opt.force,
print_names=opt.print)
elif opt.auto or opt.force:
sys.stdout.flush()
path = opt.dir or git.repo(b'objects/pack')
debug1('midx: scanning %s\n' % path_msg(path))
do_midx_dir(path,
opt.output,
byte_stream(sys.stdout),
auto=opt.auto,
force=opt.force,
max_files=opt.max_files)
else:
o.fatal("you must use -f or -a or provide input filenames")
if saved_errors:
log('WARNING: %d errors encountered.\n' % len(saved_errors))
sys.exit(1)
def ex(cmd, **kwargs):
"""Print cmd to stderr and then run it as per ex(...).
Print the subprocess stderr to stderr if stderr=PIPE and there's
any data.
"""
logcmd(cmd)
result = run(cmd, **kwargs)
if result.err:
sys.stderr.flush()
byte_stream(sys.stderr).write(result.err)
return result
def main(argv):
global repo
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
git.check_repo_or_die()
sys.stdout.flush()
out = byte_stream(sys.stdout)
stdin = byte_stream(sys.stdin)
with LocalRepo() as repo:
present_interface(stdin, out, extra, repo)
if saved_errors:
log('warning: %d errors encountered\n' % len(saved_errors))
sys.exit(1)
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
git.check_repo_or_die()
tags = [t for sublist in git.tags().values() for t in sublist]
if opt.delete:
# git.delete_ref() doesn't complain if a ref doesn't exist. We
# could implement this verification but we'd need to read in the
# contents of the tag file and pass the hash, and we already know
# about the tag's existance via "tags".
tag_name = argv_bytes(opt.delete)
if not opt.force and tag_name not in tags:
log("error: tag '%s' doesn't exist\n" % path_msg(tag_name))
sys.exit(1)
tag_file = b'refs/tags/%s' % tag_name
git.delete_ref(tag_file)
sys.exit(0)
if not extra:
for t in tags:
sys.stdout.flush()
out = byte_stream(sys.stdout)
out.write(t)
out.write(b'\n')
sys.exit(0)
elif len(extra) != 2:
o.fatal('expected commit ref and hash')
tag_name, commit = map(argv_bytes, extra[:2])
if not tag_name:
o.fatal("tag name must not be empty.")
debug1("args: tag name = %s; commit = %s\n" %
(path_msg(tag_name), commit.decode('ascii')))
if tag_name in tags and not opt.force:
log("bup: error: tag '%s' already exists\n" % path_msg(tag_name))
sys.exit(1)
if tag_name.startswith(b'.'):
o.fatal("'%s' is not a valid tag name." % path_msg(tag_name))
try:
hash = git.rev_parse(commit)
except git.GitError as e:
log("bup: error: %s" % e)
sys.exit(2)
if not hash:
log("bup: error: commit %s not found.\n" % commit.decode('ascii'))
sys.exit(2)
with git.PackIdxList(git.repo(b'objects/pack')) as pL:
if not pL.exists(hash):
log("bup: error: commit %s not found.\n" % commit.decode('ascii'))
sys.exit(2)
git.update_ref(b'refs/tags/' + tag_name, hash, None, force=True)
def check_prune_result(expected):
actual = sorted([int(x)
for x in exo([b'git', b'log',
b'--pretty=format:%at']).out.splitlines()])
if expected != actual:
for x in expected:
print('ex:', x, strftime('%Y-%m-%d-%H%M%S', localtime(x)),
file=stderr)
for line in diff_bytes(unified_diff,
[result_diffline(x) for x in expected],
[result_diffline(x) for x in actual],
fromfile=b'expected', tofile=b'actual'):
sys.stderr.flush()
byte_stream(sys.stderr).write(line)
wvpass(expected == actual)
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if extra:
o.fatal("no arguments expected")
git.check_repo_or_die()
with git.PackIdxList(git.repo(b'objects/pack'),
ignore_midx=opt.ignore_midx) as mi:
def do_predict(ix, out):
total = len(ix)
maxdiff = 0
for count,i in enumerate(ix):
prefix = struct.unpack('!Q', i[:8])[0]
expected = prefix * total // (1 << 64)
diff = count - expected
maxdiff = max(maxdiff, abs(diff))
out.write(b'%d of %d (%.3f%%) '
% (maxdiff, len(ix), maxdiff * 100.0 / len(ix)))
out.flush()
assert(count+1 == len(ix))
sys.stdout.flush()
out = byte_stream(sys.stdout)
if opt.predict:
if opt.ignore_midx:
for pack in mi.packs:
do_predict(pack, out)
else:
do_predict(mi, out)
else:
# default mode: find longest matching prefix
last = b'\0'*20
longmatch = 0
for i in mi:
if i == last:
continue
#assert(str(i) >= last)
pm = _helpers.bitmatch(last, i)
longmatch = max(longmatch, pm)
last = i
out.write(b'%d\n' % longmatch)
log('%d matching prefix bits\n' % longmatch)
doublings = math.log(len(mi), 2)
bpd = longmatch / doublings
log('%.2f bits per doubling\n' % bpd)
remain = 160 - longmatch
rdoublings = remain / bpd
log('%d bits (%.2f doublings) remaining\n' % (remain, rdoublings))
larger = 2**rdoublings
log('%g times larger is possible\n' % larger)
perperson = larger/POPULATION_OF_EARTH
log('\nEveryone on earth could have %d data sets like yours, all in one\n'
'repository, and we would expect 1 object collision.\n'
% int(perperson))
def __init__(self, infd, outp):
BaseConn.__init__(self, outp)
# Anything that comes through before the sync string was not
# multiplexed and can be assumed to be debug/log before mux init.
tail = b''
while tail != b'BUPMUX':
b = os.read(infd, (len(tail) < 6) and (6-len(tail)) or 1)
if not b:
raise IOError('demux: unexpected EOF during initialization')
tail += b
byte_stream(sys.stderr).write(tail[:-6]) # pre-mux log messages
tail = tail[-6:]
self.infd = infd
self.reader = None
self.buf = None
self.closed = False
def _next_packet(self, timeout):
if self.closed: return False
rl, wl, xl = select.select([self.infd], [], [], timeout)
if not rl: return False
assert(rl[0] == self.infd)
ns = b''.join(checked_reader(self.infd, 5))
n, fdw = struct.unpack('!IB', ns)
assert(n <= MAX_PACKET)
if fdw == 1:
self.reader = checked_reader(self.infd, n)
elif fdw == 2:
for buf in checked_reader(self.infd, n):
byte_stream(sys.stderr).write(buf)
elif fdw == 3:
self.closed = True
debug2("DemuxConn: marked closed\n")
return True
def main(argv):
git.check_repo_or_die()
sys.stdout.flush()
out = byte_stream(sys.stdout)
# Check out lib/bup/ls.py for the opt spec
rc = ls.via_cmdline(argv[1:], out=out)
sys.exit(rc)
def runtest(self):
cmd = str(self.fspath)
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out = p.communicate()[0]
sys.stdout.flush()
byte_stream(sys.stdout).write(out)
failures = [
line for line in out.splitlines()
if (line.startswith(b'!') and line.lower().endswith(b' failed'))
]
if failures or p.returncode != 0:
raise BupSubprocFailure(
'%s failed (exit %d, %d failures)' %
(cmd, p.returncode, len(failures)), cmd, p.returncode,
failures)
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if extra:
o.fatal('no arguments expected')
git.check_repo_or_die()
sys.stdout.flush()
out = byte_stream(sys.stdout)
report(-1, out)
_helpers.random_sha()
report(0, out)
with git.PackIdxList(git.repo(b'objects/pack'),
ignore_midx=opt.ignore_midx) as m:
if opt.existing:
def foreverit(mi):
while 1:
for e in mi:
yield e
objit = iter(foreverit(m))
for c in range(opt.cycles):
for n in range(opt.number):
if opt.existing:
bin = next(objit)
assert (m.exists(bin))
else:
bin = _helpers.random_sha()
# technically, a randomly generated object id might exist.
# but the likelihood of that is the likelihood of finding
# a collision in sha-1 by accident, which is so unlikely that
# we don't care.
assert (not m.exists(bin))
report((c + 1) * opt.number, out)
if bloom._total_searches:
out.write(
b'bloom: %d objects searched in %d steps: avg %.3f steps/object\n'
% (bloom._total_searches, bloom._total_steps,
bloom._total_steps * 1.0 / bloom._total_searches))
if midx._total_searches:
out.write(
b'midx: %d objects searched in %d steps: avg %.3f steps/object\n' %
(midx._total_searches, midx._total_steps,
midx._total_steps * 1.0 / midx._total_searches))
if git._total_searches:
out.write(
b'idx: %d objects searched in %d steps: avg %.3f steps/object\n' %
(git._total_searches, git._total_steps,
git._total_steps * 1.0 / git._total_searches))
out.write(b'Total time: %.3fs\n' % (time.time() - start))
def run_subproc_cmd(args):
c = (do_profile and [sys.executable, b'-m', b'cProfile'] or []) + args
if not (fix_stdout or fix_stderr):
os.execvp(c[0], c)
sys.stdout.flush()
sys.stderr.flush()
out = byte_stream(sys.stdout)
err = byte_stream(sys.stderr)
p = None
try:
p = subprocess.Popen(c,
stdout=PIPE if fix_stdout else out,
stderr=PIPE if fix_stderr else err,
env=tty_env,
bufsize=4096,
close_fds=True)
# Assume p will receive these signals and quit, which will
# then cause us to quit.
for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT):
signal.signal(sig, signal.SIG_IGN)
srcs = []
dests = []
if fix_stdout:
srcs.append(p.stdout.fileno())
dests.append(out.fileno())
if fix_stderr:
srcs.append(p.stderr.fileno())
dests.append(err.fileno())
filter_output(srcs, dests)
return p.wait()
except BaseException as ex:
add_ex_tb(ex)
try:
if p and p.poll() == None:
os.kill(p.pid, signal.SIGTERM)
p.wait()
except BaseException as kill_ex:
raise add_ex_ctx(add_ex_tb(kill_ex), ex)
raise ex
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
git.check_repo_or_die()
if not extra:
o.fatal('must specify a target')
if len(extra) > 1:
o.fatal('only one target file allowed')
if opt.bupm and opt.meta:
o.fatal('--meta and --bupm are incompatible')
target = argv_bytes(extra[0])
if not re.match(br'/*[^/]+/[^/]+', target):
o.fatal("path %r doesn't include a branch and revision" % target)
with LocalRepo() as repo:
resolved = vfs.resolve(repo, target, follow=False)
leaf_name, leaf_item = resolved[-1]
if not leaf_item:
log('error: cannot access %r in %r\n' %
(b'/'.join(name for name, item in resolved), target))
sys.exit(1)
mode = vfs.item_mode(leaf_item)
sys.stdout.flush()
out = byte_stream(sys.stdout)
if opt.bupm:
if not stat.S_ISDIR(mode):
o.fatal('%r is not a directory' % target)
_, bupm_oid = vfs.tree_data_and_bupm(repo, leaf_item.oid)
if bupm_oid:
with vfs.tree_data_reader(repo, bupm_oid) as meta_stream:
out.write(meta_stream.read())
elif opt.meta:
augmented = vfs.augment_item_meta(repo,
leaf_item,
include_size=True)
out.write(augmented.meta.encode())
else:
if stat.S_ISREG(mode):
with vfs.fopen(repo, leaf_item) as f:
for b in chunkyreader(f):
out.write(b)
else:
o.fatal('%r is not a plain file' % target)
if saved_errors:
log('warning: %d errors encountered\n' % len(saved_errors))
sys.exit(1)
def runtest(self):
cmd = str(self.fspath)
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out = p.communicate()[0]
sys.stdout.flush()
byte_stream(sys.stdout).write(out)
lines = out.splitlines()
for line in lines:
if line.startswith(b'!') and line.lower().endswith(b' skip ok'):
pytest.skip(line.decode('ascii'))
return
failures = [line for line in lines
if (line.startswith(b'!')
and line.lower().endswith(b' failed'))]
if b'AssertionError' in out:
raise BupSubprocFailure('AssertionError detected')
if failures or p.returncode != 0:
raise BupSubprocFailure('%s failed (exit %d, %d failures)'
% (cmd, p.returncode, len(failures)),
cmd, p.returncode, failures)
def _next_packet(self, timeout):
if self.closed: return False
rl, wl, xl = select.select([self.infd], [], [], timeout)
if not rl: return False
assert (rl[0] == self.infd)
ns = b''.join(checked_reader(self.infd, 5))
n, fdw = struct.unpack('!IB', ns)
if n > MAX_PACKET:
# assume that something went wrong and print stuff
ns += os.read(self.infd, 1024)
stderr = byte_stream(sys.stderr)
stderr.write(ns)
stderr.flush()
raise Exception("Connection broken")
if fdw == 1:
self.reader = checked_reader(self.infd, n)
elif fdw == 2:
for buf in checked_reader(self.infd, n):
byte_stream(sys.stderr).write(buf)
elif fdw == 3:
self.closed = True
debug2("DemuxConn: marked closed\n")
return True
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
handle_ctrl_c()
opt.find = argv_bytes(opt.find) if opt.find else b''
if not extra:
o.fatal('you must provide at least one filename')
if len(opt.find) > 40:
o.fatal('--find parameter must be <= 40 chars long')
else:
if len(opt.find) % 2:
s = opt.find + b'0'
else:
s = opt.find
try:
bin = unhexlify(s)
except TypeError:
o.fatal('--find parameter is not a valid hex string')
sys.stdout.flush()
out = byte_stream(sys.stdout)
find = opt.find.lower()
count = 0
idxfiles = [argv_bytes(x) for x in extra]
for name in idxfiles:
try:
ix = git.open_idx(name)
except git.GitError as e:
add_error('%r: %s' % (name, e))
continue
if len(opt.find) == 40:
if ix.exists(bin):
out.write(b'%s %s\n' % (name, find))
else:
# slow, exhaustive search
for _i in ix:
i = hexlify(_i)
if i.startswith(find):
out.write(b'%s %s\n' % (name, i))
qprogress('Searching: %d\r' % count)
count += 1
if saved_errors:
log('WARNING: %d errors encountered while saving.\n' %
len(saved_errors))
sys.exit(1)
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if extra:
o.fatal('no arguments expected')
# get the subcommand's argv.
# Normally we could just pass this on the command line, but since we'll often
# be getting called on the other end of an ssh pipe, which tends to mangle
# argv (by sending it via the shell), this way is much safer.
stdin = byte_stream(sys.stdin)
buf = stdin.read(4)
sz = struct.unpack('!I', buf)[0]
assert(sz > 0)
assert(sz < 1000000)
buf = stdin.read(sz)
assert(len(buf) == sz)
argv = buf.split(b'\0')
argv[0] = path.exe()
argv = [argv[0], b'mux', b'--'] + argv
# stdin/stdout are supposedly connected to 'bup server' that the caller
# started for us (often on the other end of an ssh tunnel), so we don't want
# to misuse them. Move them out of the way, then replace stdout with
# a pointer to stderr in case our subcommand wants to do something with it.
#
# It might be nice to do the same with stdin, but my experiments showed that
# ssh seems to make its child's stderr a readable-but-never-reads-anything
# socket. They really should have used shutdown(SHUT_WR) on the other end
# of it, but probably didn't. Anyway, it's too messy, so let's just make sure
# anyone reading from stdin is disappointed.
#
# (You can't just leave stdin/stdout "not open" by closing the file
# descriptors. Then the next file that opens is automatically assigned 0 or 1,
# and people *trying* to read/write stdin/stdout get screwed.)
os.dup2(0, 3)
os.dup2(1, 4)
os.dup2(2, 1)
fd = os.open(os.devnull, os.O_RDONLY)
os.dup2(fd, 0)
os.close(fd)
environ[b'BUP_SERVER_REVERSE'] = helpers.hostname()
os.execvp(argv[0], argv)
sys.exit(99)
def main(argv):
# Give the subcommand exclusive access to stdin.
orig_stdin = os.dup(0)
devnull = os.open(os.devnull, os.O_RDONLY)
os.dup2(devnull, 0)
os.close(devnull)
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if len(extra) < 1:
o.fatal('command is required')
subcmd = extra
debug2('bup mux: starting %r\n' % (extra, ))
outr, outw = os.pipe()
errr, errw = os.pipe()
def close_fds():
os.close(outr)
os.close(errr)
p = subprocess.Popen(subcmd,
stdin=orig_stdin,
stdout=outw,
stderr=errw,
close_fds=False,
preexec_fn=close_fds)
os.close(outw)
os.close(errw)
sys.stdout.flush()
out = byte_stream(sys.stdout)
out.write(b'BUPMUX')
out.flush()
mux(p, out.fileno(), outr, errr)
os.close(outr)
os.close(errr)
prv = p.wait()
if prv:
debug1('%s exited with code %d\n' % (extra[0], prv))
debug1('bup mux: done\n')
sys.exit(prv)
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
total = (opt.date or 0) + (opt.commit or 0)
if total > 1:
o.fatal('at most one option expected')
sys.stdout.flush()
out = byte_stream(sys.stdout)
if opt.date:
out.write(version.date.split(b' ')[0] + b'\n')
elif opt.commit:
out.write(version.commit + b'\n')
else:
out.write(version.version + b'\n')
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
sys.stdout.flush()
out = byte_stream(sys.stdout)
out.write(b'bup %s\n' % version.version)
out.write(b'Source %s %s\n' % (version.commit, version.date))
have_readline = getattr(_helpers, 'readline', None)
have_libacl = getattr(_helpers, 'read_acl', None)
have_xattr = metadata.xattr
out.write(b' Python: %s\n' % platform.python_version().encode('ascii'))
show_support(out, have_readline, b'Command line editing (e.g. bup ftp)')
show_support(out, have_libacl, b'Saving and restoring POSIX ACLs')
show_support(out, have_xattr,
b'Saving and restoring extended attributes (xattrs)')
def __init__(self, infd, outp):
BaseConn.__init__(self, outp)
# Anything that comes through before the sync string was not
# multiplexed and can be assumed to be debug/log before mux init.
stderr = byte_stream(sys.stderr)
cookie = b'BUPMUX'
pos = 0
while True:
b = os.read(infd, 1)
# Make sure to write all pre-BUPMUX output to stderr
if not b:
ex = IOError('demux: unexpected EOF during initialization')
with pending_raise(ex):
stderr.write(cookie[:pos])
stderr.flush()
if b == bytes_from_byte(cookie[pos]):
pos += 1
if pos == len(cookie):
break
continue
# If we can't find a new char of 'BUPMUX' then we must have some
# pre-mux log messages - output those as soon and as complete as
# possible.
#
# \r\n interacts badly with print_clean_line() in the main bup
# so remove all the \r so we see the full the lines. This assumes
# that nothing at this point will intentionally delete lines, but
# as we're just during SSH init that seems reasonable.
if b == b'\r':
continue
stderr.write(cookie[:pos]) # could be we have "BU" in the logs or so
pos = 0
stderr.write(b) # pre-mux log messages
stderr.flush()
self.infd = infd
self.reader = None
self.buf = None
self.closed = False
def __init__(self, infd, outp):
BaseConn.__init__(self, outp)
# Anything that comes through before the sync string was not
# multiplexed and can be assumed to be debug/log before mux init.
tail = b''
stderr = byte_stream(sys.stderr)
while tail != b'BUPMUX':
# Make sure to write all pre-BUPMUX output to stderr
b = os.read(infd, (len(tail) < 6) and (6 - len(tail)) or 1)
if not b:
ex = IOError('demux: unexpected EOF during initialization')
with pending_raise(ex):
stderr.write(tail)
stderr.flush()
tail += b
stderr.write(tail[:-6])
tail = tail[-6:]
stderr.flush()
self.infd = infd
self.reader = None
self.buf = None
self.closed = False
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if len(extra) != 1:
o.fatal("exactly one filename expected")
drecurse_top = argv_bytes(extra[0])
excluded_paths = parse_excludes(flags, o.fatal)
if not drecurse_top.startswith(b'/'):
excluded_paths = [relpath(x) for x in excluded_paths]
exclude_rxs = parse_rx_excludes(flags, o.fatal)
it = drecurse.recursive_dirlist([drecurse_top],
opt.xdev,
excluded_paths=excluded_paths,
exclude_rxs=exclude_rxs)
if opt.profile:
import cProfile
def do_it():
for i in it:
pass
cProfile.run('do_it()')
else:
if opt.quiet:
for i in it:
pass
else:
sys.stdout.flush()
out = byte_stream(sys.stdout)
for (name, st) in it:
out.write(name + b'\n')
if saved_errors:
log('WARNING: %d errors encountered.\n' % len(saved_errors))
sys.exit(1)
def main(argv):
opt = opts_from_cmdline(argv)
if opt.verbose >= 2:
git.verbose = opt.verbose - 1
if opt.fanout:
hashsplit.fanout = opt.fanout
if opt.blobs:
hashsplit.fanout = 0
if opt.bwlimit:
client.bwlimit = opt.bwlimit
start_time = time.time()
sys.stdout.flush()
out = byte_stream(sys.stdout)
stdin = byte_stream(sys.stdin)
if opt.git_ids:
# the input is actually a series of git object ids that we should retrieve
# and split.
#
# This is a bit messy, but basically it converts from a series of
# CatPipe.get() iterators into a series of file-type objects.
# It would be less ugly if either CatPipe.get() returned a file-like object
# (not very efficient), or split_to_shalist() expected an iterator instead
# of a file.
cp = git.CatPipe()
class IterToFile:
def __init__(self, it):
self.it = iter(it)
def read(self, size):
v = next(self.it, None)
return v or b''
def read_ids():
while 1:
line = stdin.readline()
if not line:
break
if line:
line = line.strip()
try:
it = cp.get(line.strip())
next(it, None) # skip the file info
except KeyError as e:
add_error('error: %s' % e)
continue
yield IterToFile(it)
files = read_ids()
else:
# the input either comes from a series of files or from stdin.
if opt.sources:
files = (open(argv_bytes(fn), 'rb') for fn in opt.sources)
else:
files = [stdin]
writing = not (opt.noop or opt.copy)
remote_dest = opt.remote or opt.is_reverse
if writing:
git.check_repo_or_die()
if remote_dest and writing:
cli = repo = client.Client(opt.remote)
else:
cli = nullcontext()
repo = git
# cli creation must be last nontrivial command in each if clause above
with cli:
if opt.name and writing:
refname = opt.name and b'refs/heads/%s' % opt.name
oldref = repo.read_ref(refname)
else:
refname = oldref = None
if not writing:
pack_writer = NoOpPackWriter()
elif not remote_dest:
pack_writer = git.PackWriter(compression_level=opt.compress,
max_pack_size=opt.max_pack_size,
max_pack_objects=opt.max_pack_objects)
else:
pack_writer = cli.new_packwriter(
compression_level=opt.compress,
max_pack_size=opt.max_pack_size,
max_pack_objects=opt.max_pack_objects)
# packwriter creation must be last command in each if clause above
with pack_writer:
commit = split(opt, files, oldref, out, pack_writer)
# pack_writer must be closed before we can update the ref
if refname:
repo.update_ref(refname, commit, oldref)
secs = time.time() - start_time
size = hashsplit.total_split
if opt.bench:
log('bup: %.2f kbytes in %.2f secs = %.2f kbytes/sec\n' %
(size / 1024, secs, size / 1024 / secs))
if saved_errors:
log('WARNING: %d errors encountered while saving.\n' %
len(saved_errors))
sys.exit(1)
pct = total and count*100.0/total or 100
progress('Saving: %.2f%% (%d/%dk, %d/%d files), done. \n'
% (pct, count/1024, total/1024, fcount, ftotal))
while len(parts) > 1: # _pop() all the parts above the root
_pop(force_tree = None)
assert(len(shalists) == 1)
assert(len(metalists) == 1)
# Finish the root directory.
tree = _pop(force_tree = None,
# When there's a collision, use empty metadata for the root.
dir_metadata = metadata.Metadata() if root_collision else None)
sys.stdout.flush()
out = byte_stream(sys.stdout)
if opt.tree:
out.write(hexlify(tree))
out.write(b'\n')
if opt.commit or name:
msg = (b'bup save\n\nGenerated by command:\n%r\n'
% [argv_bytes(x) for x in sys.argv])
userline = (b'%s <%s@%s>' % (userfullname(), username(), hostname()))
commit = w.new_commit(tree, oldref, userline, date, None,
userline, date, None, msg)
if opt.commit:
out.write(hexlify(commit))
out.write(b'\n')
msr.close()
optspec = """
bup on--server
--
This command is run automatically by 'bup on'
"""
o = options.Options(optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])
if extra:
o.fatal('no arguments expected')
# get the subcommand's argv.
# Normally we could just pass this on the command line, but since we'll often
# be getting called on the other end of an ssh pipe, which tends to mangle
# argv (by sending it via the shell), this way is much safer.
stdin = byte_stream(sys.stdin)
buf = stdin.read(4)
sz = struct.unpack('!I', buf)[0]
assert(sz > 0)
assert(sz < 1000000)
buf = stdin.read(sz)
assert(len(buf) == sz)
argv = buf.split(b'\0')
argv[0] = path.exe()
argv = [argv[0], b'mux', b'--'] + argv
# stdin/stdout are supposedly connected to 'bup server' that the caller
# started for us (often on the other end of an ssh tunnel), so we don't want
# to misuse them. Move them out of the way, then replace stdout with
# a pointer to stderr in case our subcommand wants to do something with it.
def main(argv):
target_filename = b''
active_fields = metadata.all_fields
o = options.Options(optspec)
(opt, flags, remainder) = o.parse_bytes(argv[1:])
atime_resolution = parse_timestamp_arg('atime', opt.atime_resolution)
mtime_resolution = parse_timestamp_arg('mtime', opt.mtime_resolution)
ctime_resolution = parse_timestamp_arg('ctime', opt.ctime_resolution)
treat_include_fields_as_definitive = True
for flag, value in flags:
if flag == '--exclude-fields':
exclude_fields = frozenset(value.split(','))
for f in exclude_fields:
if not f in metadata.all_fields:
o.fatal(f + ' is not a valid field name')
active_fields = active_fields - exclude_fields
treat_include_fields_as_definitive = False
elif flag == '--include-fields':
include_fields = frozenset(value.split(','))
for f in include_fields:
if not f in metadata.all_fields:
o.fatal(f + ' is not a valid field name')
if treat_include_fields_as_definitive:
active_fields = include_fields
treat_include_fields_as_definitive = False
else:
active_fields = active_fields | include_fields
opt.verbose = opt.verbose or 0
opt.quiet = opt.quiet or 0
metadata.verbose = opt.verbose - opt.quiet
sys.stdout.flush()
out = byte_stream(sys.stdout)
first_path = True
for path in remainder:
path = argv_bytes(path)
try:
m = metadata.from_path(path, archive_path=path)
except (OSError, IOError) as e:
if e.errno == errno.ENOENT:
add_error(e)
continue
else:
raise
if metadata.verbose >= 0:
if not first_path:
out.write(b'\n')
if atime_resolution != 1:
m.atime = (m.atime / atime_resolution) * atime_resolution
if mtime_resolution != 1:
m.mtime = (m.mtime / mtime_resolution) * mtime_resolution
if ctime_resolution != 1:
m.ctime = (m.ctime / ctime_resolution) * ctime_resolution
out.write(metadata.detailed_bytes(m, active_fields))
out.write(b'\n')
first_path = False
if saved_errors:
log('WARNING: %d errors encountered.\n' % len(saved_errors))
sys.exit(1)
else:
sys.exit(0)
