def main(argv):
o = options.Options(optspec, optfunc=getopt.getopt)
opt, flags, extra = o.parse_bytes(argv[1:])
host = opt.listen
port = opt.port and int(opt.port) or 1982
socks = []
e = None
for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC,
socket.SOCK_STREAM, 0, socket.AI_PASSIVE):
af, socktype, proto, canonname, sa = res
try:
s = socket.socket(af, socktype, proto)
except socket.error as e:
continue
try:
if af == socket.AF_INET6:
log("bup daemon: listening on [%s]:%s\n" % sa[:2])
else:
log("bup daemon: listening on %s:%s\n" % sa[:2])
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(sa)
s.listen(1)
fcntl.fcntl(s.fileno(), fcntl.F_SETFD, fcntl.FD_CLOEXEC)
except socket.error as e:
s.close()
continue
socks.append(s)
if not socks:
log('bup daemon: listen socket: %s\n' % e.args[1])
sys.exit(1)
try:
while True:
[rl, wl, xl] = select.select(socks, [], [], 60)
for l in rl:
s, src = l.accept()
try:
log("Socket accepted connection from %s\n" % (src, ))
fd1 = os.dup(s.fileno())
fd2 = os.dup(s.fileno())
s.close()
sp = subprocess.Popen(
[path.exe(), 'mux', '--',
path.exe(), 'server'] + extra,
stdin=fd1,
stdout=fd2)
finally:
os.close(fd1)
os.close(fd2)
finally:
for l in socks:
l.shutdown(socket.SHUT_RDWR)
l.close()
debug1("bup daemon: done")
def test_midx_refreshing(tmpdir):
environ[b'BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
c = client.Client(bupdir, create=True)
rw = c.new_packwriter()
rw.new_blob(s1)
p1base = rw.breakpoint()
p1name = os.path.join(c.cachedir, p1base)
s1sha = rw.new_blob(s1) # should not be written; it's already in p1
s2sha = rw.new_blob(s2)
p2base = rw.close()
p2name = os.path.join(c.cachedir, p2base)
del rw
pi = git.PackIdxList(bupdir + b'/objects/pack')
assert len(pi.packs) == 2
pi.refresh()
assert len(pi.packs) == 2
assert sorted([os.path.basename(i.name)
for i in pi.packs]) == sorted([p1base, p2base])
p1 = git.open_idx(p1name)
assert p1.exists(s1sha)
p2 = git.open_idx(p2name)
assert not p2.exists(s1sha)
assert p2.exists(s2sha)
subprocess.call([path.exe(), b'midx', b'-f'])
pi.refresh()
assert len(pi.packs) == 1
pi.refresh(skip_midx=True)
assert len(pi.packs) == 2
pi.refresh(skip_midx=False)
assert len(pi.packs) == 1
def connect(rhost, port, subcmd, stderr=None):
"""Connect to 'rhost' and execute the bup subcommand 'subcmd' on it."""
assert not re.search(br'[^\w-]', subcmd)
if rhost is None or rhost == b'-':
argv = [path.exe(), subcmd]
else:
buglvl = helpers.atoi(environ.get(b'BUP_DEBUG'))
force_tty = helpers.atoi(environ.get(b'BUP_FORCE_TTY'))
cmd = b"""
sh -c 'BUP_DEBUG=%d BUP_FORCE_TTY=%d bup %s'
""" % (buglvl, force_tty, subcmd)
argv = [b'ssh']
if port:
argv.extend((b'-p', port))
argv.extend((rhost, b'--', cmd.strip()))
#helpers.log('argv is: %r\n' % argv)
if sys.version_info[0] < 3:
return subprocess.Popen(argv,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=stderr,
preexec_fn=lambda: os.setsid())
else:
return subprocess.Popen(argv,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=stderr,
start_new_session=True)
def auto_midx(objdir):
args = [path.exe(), 'midx', '--auto', '--dir', objdir]
try:
rv = subprocess.call(args, stdout=open('/dev/null', 'w'))
except OSError, e:
# make sure 'args' gets printed to help with debugging
add_error('%r: exception: %s' % (args, e))
raise
def auto_midx(objdir):
args = [path.exe(), b'midx', b'--auto', b'--dir', objdir]
try:
rv = subprocess.call(args, stdout=open(os.devnull, 'w'))
except OSError as e:
# make sure 'args' gets printed to help with debugging
add_error('%r: exception: %s' % (args, e))
raise
if rv:
add_error('%r: returned %d' % (args, rv))
args = [path.exe(), b'bloom', b'--dir', objdir]
try:
rv = subprocess.call(args, stdout=open(os.devnull, 'w'))
except OSError as e:
# make sure 'args' gets printed to help with debugging
add_error('%r: exception: %s' % (args, e))
raise
if rv:
add_error('%r: returned %d' % (args, rv))
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if len(extra) == 0:
# the wrapper program provides the default usage string
os.execvp(path.exe(), [path.exe()])
elif len(extra) == 1:
docname = (extra[0] == 'bup' and b'bup'
or (b'bup-%s' % argv_bytes(extra[0])))
manpath = os.path.join(path.exedir(),
b'../../Documentation/' + docname + b'.[1-9]')
g = glob.glob(manpath)
try:
if g:
os.execvp('man', ['man', '-l', g[0]])
else:
os.execvp('man', ['man', docname])
except OSError as e:
sys.stderr.write('Unable to run man command: %s\n' % e)
sys.exit(1)
else:
o.fatal("exactly one command name expected")
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if extra:
o.fatal('no arguments expected')
# get the subcommand's argv.
# Normally we could just pass this on the command line, but since we'll often
# be getting called on the other end of an ssh pipe, which tends to mangle
# argv (by sending it via the shell), this way is much safer.
stdin = byte_stream(sys.stdin)
buf = stdin.read(4)
sz = struct.unpack('!I', buf)[0]
assert(sz > 0)
assert(sz < 1000000)
buf = stdin.read(sz)
assert(len(buf) == sz)
argv = buf.split(b'\0')
argv[0] = path.exe()
argv = [argv[0], b'mux', b'--'] + argv
# stdin/stdout are supposedly connected to 'bup server' that the caller
# started for us (often on the other end of an ssh tunnel), so we don't want
# to misuse them. Move them out of the way, then replace stdout with
# a pointer to stderr in case our subcommand wants to do something with it.
#
# It might be nice to do the same with stdin, but my experiments showed that
# ssh seems to make its child's stderr a readable-but-never-reads-anything
# socket. They really should have used shutdown(SHUT_WR) on the other end
# of it, but probably didn't. Anyway, it's too messy, so let's just make sure
# anyone reading from stdin is disappointed.
#
# (You can't just leave stdin/stdout "not open" by closing the file
# descriptors. Then the next file that opens is automatically assigned 0 or 1,
# and people *trying* to read/write stdin/stdout get screwed.)
os.dup2(0, 3)
os.dup2(1, 4)
os.dup2(2, 1)
fd = os.open(os.devnull, os.O_RDONLY)
os.dup2(fd, 0)
os.close(fd)
environ[b'BUP_SERVER_REVERSE'] = helpers.hostname()
os.execvp(argv[0], argv)
sys.exit(99)
def test_midx_refreshing():
with no_lingering_errors():
with test_tempdir(b'bup-tclient-') as tmpdir:
environ[b'BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
c = client.Client(bupdir, create=True)
rw = c.new_packwriter()
rw.new_blob(s1)
p1base = rw.breakpoint()
p1name = os.path.join(c.cachedir, p1base)
s1sha = rw.new_blob(
s1) # should not be written; it's already in p1
s2sha = rw.new_blob(s2)
p2base = rw.close()
p2name = os.path.join(c.cachedir, p2base)
del rw
pi = git.PackIdxList(bupdir + b'/objects/pack')
WVPASSEQ(len(pi.packs), 2)
pi.refresh()
WVPASSEQ(len(pi.packs), 2)
WVPASSEQ(sorted([os.path.basename(i.name) for i in pi.packs]),
sorted([p1base, p2base]))
p1 = git.open_idx(p1name)
WVPASS(p1.exists(s1sha))
p2 = git.open_idx(p2name)
WVFAIL(p2.exists(s1sha))
WVPASS(p2.exists(s2sha))
subprocess.call([path.exe(), b'midx', b'-f'])
pi.refresh()
WVPASSEQ(len(pi.packs), 1)
pi.refresh(skip_midx=True)
WVPASSEQ(len(pi.packs), 2)
pi.refresh(skip_midx=False)
WVPASSEQ(len(pi.packs), 1)
def main(argv):
o = options.Options(optspec, optfunc=getopt.getopt)
opt, flags, extra = o.parse_bytes(argv[1:])
if len(extra) < 2:
o.fatal('arguments expected')
class SigException(Exception):
def __init__(self, signum):
self.signum = signum
Exception.__init__(self, 'signal %d received' % signum)
def handler(signum, frame):
raise SigException(signum)
signal.signal(signal.SIGTERM, handler)
signal.signal(signal.SIGINT, handler)
sys.stdout.flush()
out = byte_stream(sys.stdout)
try:
sp = None
p = None
ret = 99
hp = argv_bytes(extra[0]).split(b':')
if len(hp) == 1:
(hostname, port) = (hp[0], None)
else:
(hostname, port) = hp
argv = [argv_bytes(x) for x in extra[1:]]
p = ssh.connect(hostname, port, b'on--server', stderr=PIPE)
try:
argvs = b'\0'.join([b'bup'] + argv)
p.stdin.write(struct.pack('!I', len(argvs)) + argvs)
p.stdin.flush()
sp = subprocess.Popen([path.exe(), b'server'],
stdin=p.stdout, stdout=p.stdin)
p.stdin.close()
p.stdout.close()
# Demultiplex remote client's stderr (back to stdout/stderr).
with DemuxConn(p.stderr.fileno(), open(os.devnull, "wb")) as dmc:
for line in iter(dmc.readline, b''):
out.write(line)
finally:
while 1:
# if we get a signal while waiting, we have to keep waiting, just
# in case our child doesn't die.
try:
ret = p.wait()
if sp:
sp.wait()
break
except SigException as e:
log('\nbup on: %s\n' % e)
os.kill(p.pid, e.signum)
ret = 84
except SigException as e:
if ret == 0:
ret = 99
log('\nbup on: %s\n' % e)
sys.exit(ret)
paths += glob.glob(repo('index-cache/*/.'))
return paths
def auto_midx(objdir):
args = [path.exe(), 'midx', '--auto', '--dir', objdir]
try:
rv = subprocess.call(args, stdout=open('/dev/null', 'w'))
except OSError, e:
# make sure 'args' gets printed to help with debugging
add_error('%r: exception: %s' % (args, e))
raise
if rv:
add_error('%r: returned %d' % (args, rv))
args = [path.exe(), 'bloom', '--dir', objdir]
try:
rv = subprocess.call(args, stdout=open('/dev/null', 'w'))
except OSError, e:
# make sure 'args' gets printed to help with debugging
add_error('%r: exception: %s' % (args, e))
raise
if rv:
add_error('%r: returned %d' % (args, rv))
def mangle_name(name, mode, gitmode):
"""Mangle a file name to present an abstract name for segmented files.
Mangled file names will have the ".bup" extension added to them. If a
file's name already ends with ".bup", a ".bupl" extension is added to
disambiguate normal files from semgmented ones.
o.fatal('no arguments expected')
# get the subcommand's argv.
# Normally we could just pass this on the command line, but since we'll often
# be getting called on the other end of an ssh pipe, which tends to mangle
# argv (by sending it via the shell), this way is much safer.
stdin = byte_stream(sys.stdin)
buf = stdin.read(4)
sz = struct.unpack('!I', buf)[0]
assert(sz > 0)
assert(sz < 1000000)
buf = stdin.read(sz)
assert(len(buf) == sz)
argv = buf.split(b'\0')
argv[0] = path.exe()
argv = [argv[0], b'mux', b'--'] + argv
# stdin/stdout are supposedly connected to 'bup server' that the caller
# started for us (often on the other end of an ssh tunnel), so we don't want
# to misuse them. Move them out of the way, then replace stdout with
# a pointer to stderr in case our subcommand wants to do something with it.
#
# It might be nice to do the same with stdin, but my experiments showed that
# ssh seems to make its child's stderr a readable-but-never-reads-anything
# socket. They really should have used shutdown(SHUT_WR) on the other end
# of it, but probably didn't. Anyway, it's too messy, so let's just make sure
# anyone reading from stdin is disappointed.
#
# (You can't just leave stdin/stdout "not open" by closing the file
b'join', b'cat-file', b'ftp', b'ls', b'margin', b'meta'
]
mode = None
if argv[0] in read_append_commands:
mode = b'read-append'
elif argv[0] in append_commands:
mode = b'append'
elif argv[0] in read_commands:
mode = b'read'
if mode is not None:
# we already put BUP_DIR into the environment, which
# is inherited here
sp = subprocess.Popen(
[path.exe(), b'server', b'--force-repo', b'--mode=' + mode],
stdin=p.stdout,
stdout=p.stdin)
p.stdin.close()
p.stdout.close()
# Demultiplex remote client's stderr (back to stdout/stderr).
dmc = DemuxConn(p.stderr.fileno(), open(os.devnull, "wb"))
for line in iter(dmc.readline, b''):
out.write(line)
finally:
while 1:
# if we get a signal while waiting, we have to keep waiting, just
# in case our child doesn't die.
try:
ret = p.wait()
if sp:
import os, glob, sys
sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..']
from bup import compat, options, path
from bup.compat import argv_bytes
optspec = """
bup help <command>
"""
o = options.Options(optspec)
opt, flags, extra = o.parse(compat.argv[1:])
if len(extra) == 0:
# the wrapper program provides the default usage string
os.execvp(path.exe(), [b'bup'])
elif len(extra) == 1:
docname = (extra[0] == 'bup' and b'bup'
or (b'bup-%s' % argv_bytes(extra[0])))
manpath = os.path.join(path.exedir(),
b'../../Documentation/' + docname + b'.[1-9]')
g = glob.glob(manpath)
try:
if g:
os.execvp('man', ['man', '-l', g[0]])
else:
os.execvp('man', ['man', docname])
except OSError as e:
sys.stderr.write('Unable to run man command: %s\n' % e)
sys.exit(1)
else:
def force_midx(objdir):
args = [path.exe(), b'midx', b'--auto', b'--dir', objdir]
check_call(args)
from __future__ import absolute_import, print_function
from binascii import hexlify, unhexlify
from subprocess import check_call
import struct, os, time
from wvtest import *
from bup import git, path
from bup.compat import bytes_from_byte, environ, range
from bup.helpers import localtime, log, mkdirp, readpipe
from buptest import no_lingering_errors, test_tempdir
bup_exe = path.exe()
def exc(*cmd):
print(repr(cmd), file=sys.stderr)
check_call(cmd)
def exo(*cmd):
print(repr(cmd), file=sys.stderr)
return readpipe(cmd)
@wvtest
def test_git_version_detection():
with no_lingering_errors():
# Test version types from git's tag history
s.close()
continue
socks.append(s)
if not socks:
log('bup daemon: listen socket: %s\n' % e.args[1])
sys.exit(1)
try:
while True:
[rl,wl,xl] = select.select(socks, [], [], 60)
for l in rl:
s, src = l.accept()
try:
log("Socket accepted connection from %s\n" % (src,))
fd1 = os.dup(s.fileno())
fd2 = os.dup(s.fileno())
s.close()
sp = subprocess.Popen([path.exe(), 'mux', '--',
path.exe(), 'server']
+ extra, stdin=fd1, stdout=fd2)
finally:
os.close(fd1)
os.close(fd2)
finally:
for l in socks:
l.shutdown(socket.SHUT_RDWR)
l.close()
debug1("bup daemon: done")
p = None
ret = 99
hp = extra[0].split(':')
if len(hp) == 1:
(hostname, port) = (hp[0], None)
else:
(hostname, port) = hp
argv = extra[1:]
p = ssh.connect(hostname, port, 'on--server', stderr=PIPE)
try:
argvs = '\0'.join(['bup'] + argv)
p.stdin.write(struct.pack('!I', len(argvs)) + argvs)
p.stdin.flush()
sp = subprocess.Popen([path.exe(), 'server'],
stdin=p.stdout, stdout=p.stdin)
p.stdin.close()
p.stdout.close()
# Demultiplex remote client's stderr (back to stdout/stderr).
dmc = DemuxConn(p.stderr.fileno(), open(os.devnull, "w"))
for line in iter(dmc.readline, ""):
sys.stdout.write(line)
finally:
while 1:
# if we get a signal while waiting, we have to keep waiting, just
# in case our child doesn't die.
try:
ret = p.wait()
if sp:
sp.wait()
socks.append(s)
if not socks:
log('bup daemon: listen socket: %s\n' % e.args[1])
sys.exit(1)
try:
while True:
[rl, wl, xl] = select.select(socks, [], [], 60)
for l in rl:
s, src = l.accept()
try:
log("Socket accepted connection from %s\n" % (src, ))
fd1 = os.dup(s.fileno())
fd2 = os.dup(s.fileno())
s.close()
sp = subprocess.Popen(
[path.exe(), 'mux', '--',
path.exe(), 'server'] + extra,
stdin=fd1,
stdout=fd2)
finally:
os.close(fd1)
os.close(fd2)
finally:
for l in socks:
l.shutdown(socket.SHUT_RDWR)
l.close()
debug1("bup daemon: done")
from os import symlink
from stat import S_IFDIR
from sys import stderr
from time import localtime, strftime
from wvtest import *
from bup import git, path, vfs
from bup.compat import environ
from bup.io import path_msg
from bup.metadata import Metadata
from bup.repo import LocalRepo, RemoteRepo
from bup.test.vfs import tree_dict
from buptest import ex, exo, no_lingering_errors, test_tempdir
bup_path = path.exe()
## The clear_cache() calls below are to make sure that the test starts
## from a known state since at the moment the cache entry for a given
## item (like a commit) can change. For example, its meta value might
## be promoted from a mode to a Metadata instance once the tree it
## refers to is traversed.
def prep_and_test_repo(name, create_repo, test_repo):
with no_lingering_errors():
with test_tempdir(b'bup-t' + name) as tmpdir:
bup_dir = tmpdir + b'/bup'
environ[b'GIT_DIR'] = bup_dir
environ[b'BUP_DIR'] = bup_dir
ex((bup_path, b'init'))
git.repodir = bup_dir
continue
socks.append(s)
if not socks:
log('bup daemon: listen socket: %s\n' % e.args[1])
sys.exit(1)
try:
while True:
[rl, wl, xl] = select.select(socks, [], [], 60)
for l in rl:
s, src = l.accept()
try:
log("Socket accepted connection from %s\n" % (src, ))
fd1 = os.dup(s.fileno())
fd2 = os.dup(s.fileno())
s.close()
sp = subprocess.Popen([path.exe(), 'mux', '--', 'server'] +
extra,
stdin=fd1,
stdout=fd2)
finally:
os.close(fd1)
os.close(fd2)
finally:
for l in socks:
l.shutdown(socket.SHUT_RDWR)
l.close()
debug1("bup daemon: done")
try:
hp = extra[0].split(':')
if len(hp) == 1:
(hostname, port) = (hp[0], None)
else:
(hostname, port) = hp
argv = extra[1:]
p = ssh.connect(hostname, port, 'on--server')
argvs = '\0'.join(['bup'] + argv)
p.stdin.write(struct.pack('!I', len(argvs)) + argvs)
p.stdin.flush()
sp = subprocess.Popen([path.exe(), 'server'],
stdin=p.stdout,
stdout=p.stdin)
p.stdin.close()
p.stdout.close()
finally:
while 1:
# if we get a signal while waiting, we have to keep waiting, just
# in case our child doesn't die.
try:
ret = p.wait()
sp.wait()
break
except SigException, e:
log('\nbup on: %s\n' % e)
def main(argv):
o = options.Options(optspec, optfunc=getopt.getopt)
opt, flags, extra = o.parse_bytes(argv[1:])
if len(extra) < 2:
o.fatal('arguments expected')
class SigException(Exception):
def __init__(self, signum):
self.signum = signum
Exception.__init__(self, 'signal %d received' % signum)
def handler(signum, frame):
raise SigException(signum)
signal.signal(signal.SIGTERM, handler)
signal.signal(signal.SIGINT, handler)
sys.stdout.flush()
out = byte_stream(sys.stdout)
try:
sp = None
p = None
ret = 99
hp = argv_bytes(extra[0]).split(b':')
if len(hp) == 1:
(hostname, port) = (hp[0], None)
else:
(hostname, port) = hp
argv = [argv_bytes(x) for x in extra[1:]]
p = ssh.connect(hostname, port, b'on--server', stderr=PIPE)
try:
argvs = b'\0'.join([b'bup'] + argv)
p.stdin.write(struct.pack('!I', len(argvs)) + argvs)
p.stdin.flush()
# for commands not listed here don't even execute the server
# (e.g. bup on <host> index ...)
cmdmodes = {
b'get': b'unrestricted',
b'save': b'append',
b'split': b'append',
b'tag': b'append',
b'join': b'read',
b'cat-file': b'read',
b'ftp': b'read',
b'ls': b'read',
b'margin': b'read',
b'meta': b'read',
}
mode = cmdmodes.get(argv[0], None)
if mode is not None:
# we already put BUP_DIR into the environment, which
# is inherited here
sp = subprocess.Popen([
path.exe(), b'server', b'--force-repo', b'--mode=' + mode
],
stdin=p.stdout,
stdout=p.stdin)
p.stdin.close()
p.stdout.close()
# Demultiplex remote client's stderr (back to stdout/stderr).
dmc = DemuxConn(p.stderr.fileno(), open(os.devnull, "wb"))
for line in iter(dmc.readline, b''):
out.write(line)
finally:
while 1:
# if we get a signal while waiting, we have to keep waiting, just
# in case our child doesn't die.
try:
ret = p.wait()
if sp:
sp.wait()
break
except SigException as e:
log('\nbup on: %s\n' % e)
os.kill(p.pid, e.signum)
ret = 84
except SigException as e:
if ret == 0:
ret = 99
log('\nbup on: %s\n' % e)
sys.exit(ret)
