def main(argv): o = options.Options(optspec, optfunc=getopt.getopt) opt, flags, extra = o.parse_bytes(argv[1:]) host = opt.listen port = opt.port and int(opt.port) or 1982 socks = [] e = None for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE): af, socktype, proto, canonname, sa = res try: s = socket.socket(af, socktype, proto) except socket.error as e: continue try: if af == socket.AF_INET6: log("bup daemon: listening on [%s]:%s\n" % sa[:2]) else: log("bup daemon: listening on %s:%s\n" % sa[:2]) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind(sa) s.listen(1) fcntl.fcntl(s.fileno(), fcntl.F_SETFD, fcntl.FD_CLOEXEC) except socket.error as e: s.close() continue socks.append(s) if not socks: log('bup daemon: listen socket: %s\n' % e.args[1]) sys.exit(1) try: while True: [rl, wl, xl] = select.select(socks, [], [], 60) for l in rl: s, src = l.accept() try: log("Socket accepted connection from %s\n" % (src, )) fd1 = os.dup(s.fileno()) fd2 = os.dup(s.fileno()) s.close() sp = subprocess.Popen( [path.exe(), 'mux', '--', path.exe(), 'server'] + extra, stdin=fd1, stdout=fd2) finally: os.close(fd1) os.close(fd2) finally: for l in socks: l.shutdown(socket.SHUT_RDWR) l.close() debug1("bup daemon: done")
def test_midx_refreshing(tmpdir): environ[b'BUP_DIR'] = bupdir = tmpdir git.init_repo(bupdir) c = client.Client(bupdir, create=True) rw = c.new_packwriter() rw.new_blob(s1) p1base = rw.breakpoint() p1name = os.path.join(c.cachedir, p1base) s1sha = rw.new_blob(s1) # should not be written; it's already in p1 s2sha = rw.new_blob(s2) p2base = rw.close() p2name = os.path.join(c.cachedir, p2base) del rw pi = git.PackIdxList(bupdir + b'/objects/pack') assert len(pi.packs) == 2 pi.refresh() assert len(pi.packs) == 2 assert sorted([os.path.basename(i.name) for i in pi.packs]) == sorted([p1base, p2base]) p1 = git.open_idx(p1name) assert p1.exists(s1sha) p2 = git.open_idx(p2name) assert not p2.exists(s1sha) assert p2.exists(s2sha) subprocess.call([path.exe(), b'midx', b'-f']) pi.refresh() assert len(pi.packs) == 1 pi.refresh(skip_midx=True) assert len(pi.packs) == 2 pi.refresh(skip_midx=False) assert len(pi.packs) == 1
def connect(rhost, port, subcmd, stderr=None): """Connect to 'rhost' and execute the bup subcommand 'subcmd' on it.""" assert not re.search(br'[^\w-]', subcmd) if rhost is None or rhost == b'-': argv = [path.exe(), subcmd] else: buglvl = helpers.atoi(environ.get(b'BUP_DEBUG')) force_tty = helpers.atoi(environ.get(b'BUP_FORCE_TTY')) cmd = b""" sh -c 'BUP_DEBUG=%d BUP_FORCE_TTY=%d bup %s' """ % (buglvl, force_tty, subcmd) argv = [b'ssh'] if port: argv.extend((b'-p', port)) argv.extend((rhost, b'--', cmd.strip())) #helpers.log('argv is: %r\n' % argv) if sys.version_info[0] < 3: return subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=stderr, preexec_fn=lambda: os.setsid()) else: return subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=stderr, start_new_session=True)
def auto_midx(objdir): args = [path.exe(), 'midx', '--auto', '--dir', objdir] try: rv = subprocess.call(args, stdout=open('/dev/null', 'w')) except OSError, e: # make sure 'args' gets printed to help with debugging add_error('%r: exception: %s' % (args, e)) raise
def auto_midx(objdir): args = [path.exe(), b'midx', b'--auto', b'--dir', objdir] try: rv = subprocess.call(args, stdout=open(os.devnull, 'w')) except OSError as e: # make sure 'args' gets printed to help with debugging add_error('%r: exception: %s' % (args, e)) raise if rv: add_error('%r: returned %d' % (args, rv)) args = [path.exe(), b'bloom', b'--dir', objdir] try: rv = subprocess.call(args, stdout=open(os.devnull, 'w')) except OSError as e: # make sure 'args' gets printed to help with debugging add_error('%r: exception: %s' % (args, e)) raise if rv: add_error('%r: returned %d' % (args, rv))
def main(argv): o = options.Options(optspec) opt, flags, extra = o.parse_bytes(argv[1:]) if len(extra) == 0: # the wrapper program provides the default usage string os.execvp(path.exe(), [path.exe()]) elif len(extra) == 1: docname = (extra[0] == 'bup' and b'bup' or (b'bup-%s' % argv_bytes(extra[0]))) manpath = os.path.join(path.exedir(), b'../../Documentation/' + docname + b'.[1-9]') g = glob.glob(manpath) try: if g: os.execvp('man', ['man', '-l', g[0]]) else: os.execvp('man', ['man', docname]) except OSError as e: sys.stderr.write('Unable to run man command: %s\n' % e) sys.exit(1) else: o.fatal("exactly one command name expected")
def main(argv): o = options.Options(optspec) opt, flags, extra = o.parse_bytes(argv[1:]) if extra: o.fatal('no arguments expected') # get the subcommand's argv. # Normally we could just pass this on the command line, but since we'll often # be getting called on the other end of an ssh pipe, which tends to mangle # argv (by sending it via the shell), this way is much safer. stdin = byte_stream(sys.stdin) buf = stdin.read(4) sz = struct.unpack('!I', buf)[0] assert(sz > 0) assert(sz < 1000000) buf = stdin.read(sz) assert(len(buf) == sz) argv = buf.split(b'\0') argv[0] = path.exe() argv = [argv[0], b'mux', b'--'] + argv # stdin/stdout are supposedly connected to 'bup server' that the caller # started for us (often on the other end of an ssh tunnel), so we don't want # to misuse them. Move them out of the way, then replace stdout with # a pointer to stderr in case our subcommand wants to do something with it. # # It might be nice to do the same with stdin, but my experiments showed that # ssh seems to make its child's stderr a readable-but-never-reads-anything # socket. They really should have used shutdown(SHUT_WR) on the other end # of it, but probably didn't. Anyway, it's too messy, so let's just make sure # anyone reading from stdin is disappointed. # # (You can't just leave stdin/stdout "not open" by closing the file # descriptors. Then the next file that opens is automatically assigned 0 or 1, # and people *trying* to read/write stdin/stdout get screwed.) os.dup2(0, 3) os.dup2(1, 4) os.dup2(2, 1) fd = os.open(os.devnull, os.O_RDONLY) os.dup2(fd, 0) os.close(fd) environ[b'BUP_SERVER_REVERSE'] = helpers.hostname() os.execvp(argv[0], argv) sys.exit(99)
def test_midx_refreshing(): with no_lingering_errors(): with test_tempdir(b'bup-tclient-') as tmpdir: environ[b'BUP_DIR'] = bupdir = tmpdir git.init_repo(bupdir) c = client.Client(bupdir, create=True) rw = c.new_packwriter() rw.new_blob(s1) p1base = rw.breakpoint() p1name = os.path.join(c.cachedir, p1base) s1sha = rw.new_blob( s1) # should not be written; it's already in p1 s2sha = rw.new_blob(s2) p2base = rw.close() p2name = os.path.join(c.cachedir, p2base) del rw pi = git.PackIdxList(bupdir + b'/objects/pack') WVPASSEQ(len(pi.packs), 2) pi.refresh() WVPASSEQ(len(pi.packs), 2) WVPASSEQ(sorted([os.path.basename(i.name) for i in pi.packs]), sorted([p1base, p2base])) p1 = git.open_idx(p1name) WVPASS(p1.exists(s1sha)) p2 = git.open_idx(p2name) WVFAIL(p2.exists(s1sha)) WVPASS(p2.exists(s2sha)) subprocess.call([path.exe(), b'midx', b'-f']) pi.refresh() WVPASSEQ(len(pi.packs), 1) pi.refresh(skip_midx=True) WVPASSEQ(len(pi.packs), 2) pi.refresh(skip_midx=False) WVPASSEQ(len(pi.packs), 1)
def main(argv): o = options.Options(optspec, optfunc=getopt.getopt) opt, flags, extra = o.parse_bytes(argv[1:]) if len(extra) < 2: o.fatal('arguments expected') class SigException(Exception): def __init__(self, signum): self.signum = signum Exception.__init__(self, 'signal %d received' % signum) def handler(signum, frame): raise SigException(signum) signal.signal(signal.SIGTERM, handler) signal.signal(signal.SIGINT, handler) sys.stdout.flush() out = byte_stream(sys.stdout) try: sp = None p = None ret = 99 hp = argv_bytes(extra[0]).split(b':') if len(hp) == 1: (hostname, port) = (hp[0], None) else: (hostname, port) = hp argv = [argv_bytes(x) for x in extra[1:]] p = ssh.connect(hostname, port, b'on--server', stderr=PIPE) try: argvs = b'\0'.join([b'bup'] + argv) p.stdin.write(struct.pack('!I', len(argvs)) + argvs) p.stdin.flush() sp = subprocess.Popen([path.exe(), b'server'], stdin=p.stdout, stdout=p.stdin) p.stdin.close() p.stdout.close() # Demultiplex remote client's stderr (back to stdout/stderr). with DemuxConn(p.stderr.fileno(), open(os.devnull, "wb")) as dmc: for line in iter(dmc.readline, b''): out.write(line) finally: while 1: # if we get a signal while waiting, we have to keep waiting, just # in case our child doesn't die. try: ret = p.wait() if sp: sp.wait() break except SigException as e: log('\nbup on: %s\n' % e) os.kill(p.pid, e.signum) ret = 84 except SigException as e: if ret == 0: ret = 99 log('\nbup on: %s\n' % e) sys.exit(ret)
paths += glob.glob(repo('index-cache/*/.')) return paths def auto_midx(objdir): args = [path.exe(), 'midx', '--auto', '--dir', objdir] try: rv = subprocess.call(args, stdout=open('/dev/null', 'w')) except OSError, e: # make sure 'args' gets printed to help with debugging add_error('%r: exception: %s' % (args, e)) raise if rv: add_error('%r: returned %d' % (args, rv)) args = [path.exe(), 'bloom', '--dir', objdir] try: rv = subprocess.call(args, stdout=open('/dev/null', 'w')) except OSError, e: # make sure 'args' gets printed to help with debugging add_error('%r: exception: %s' % (args, e)) raise if rv: add_error('%r: returned %d' % (args, rv)) def mangle_name(name, mode, gitmode): """Mangle a file name to present an abstract name for segmented files. Mangled file names will have the ".bup" extension added to them. If a file's name already ends with ".bup", a ".bupl" extension is added to disambiguate normal files from semgmented ones.
o.fatal('no arguments expected') # get the subcommand's argv. # Normally we could just pass this on the command line, but since we'll often # be getting called on the other end of an ssh pipe, which tends to mangle # argv (by sending it via the shell), this way is much safer. stdin = byte_stream(sys.stdin) buf = stdin.read(4) sz = struct.unpack('!I', buf)[0] assert(sz > 0) assert(sz < 1000000) buf = stdin.read(sz) assert(len(buf) == sz) argv = buf.split(b'\0') argv[0] = path.exe() argv = [argv[0], b'mux', b'--'] + argv # stdin/stdout are supposedly connected to 'bup server' that the caller # started for us (often on the other end of an ssh tunnel), so we don't want # to misuse them. Move them out of the way, then replace stdout with # a pointer to stderr in case our subcommand wants to do something with it. # # It might be nice to do the same with stdin, but my experiments showed that # ssh seems to make its child's stderr a readable-but-never-reads-anything # socket. They really should have used shutdown(SHUT_WR) on the other end # of it, but probably didn't. Anyway, it's too messy, so let's just make sure # anyone reading from stdin is disappointed. # # (You can't just leave stdin/stdout "not open" by closing the file
b'join', b'cat-file', b'ftp', b'ls', b'margin', b'meta' ] mode = None if argv[0] in read_append_commands: mode = b'read-append' elif argv[0] in append_commands: mode = b'append' elif argv[0] in read_commands: mode = b'read' if mode is not None: # we already put BUP_DIR into the environment, which # is inherited here sp = subprocess.Popen( [path.exe(), b'server', b'--force-repo', b'--mode=' + mode], stdin=p.stdout, stdout=p.stdin) p.stdin.close() p.stdout.close() # Demultiplex remote client's stderr (back to stdout/stderr). dmc = DemuxConn(p.stderr.fileno(), open(os.devnull, "wb")) for line in iter(dmc.readline, b''): out.write(line) finally: while 1: # if we get a signal while waiting, we have to keep waiting, just # in case our child doesn't die. try: ret = p.wait() if sp:
import os, glob, sys sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, options, path from bup.compat import argv_bytes optspec = """ bup help <command> """ o = options.Options(optspec) opt, flags, extra = o.parse(compat.argv[1:]) if len(extra) == 0: # the wrapper program provides the default usage string os.execvp(path.exe(), [b'bup']) elif len(extra) == 1: docname = (extra[0] == 'bup' and b'bup' or (b'bup-%s' % argv_bytes(extra[0]))) manpath = os.path.join(path.exedir(), b'../../Documentation/' + docname + b'.[1-9]') g = glob.glob(manpath) try: if g: os.execvp('man', ['man', '-l', g[0]]) else: os.execvp('man', ['man', docname]) except OSError as e: sys.stderr.write('Unable to run man command: %s\n' % e) sys.exit(1) else:
def force_midx(objdir): args = [path.exe(), b'midx', b'--auto', b'--dir', objdir] check_call(args)
from __future__ import absolute_import, print_function from binascii import hexlify, unhexlify from subprocess import check_call import struct, os, time from wvtest import * from bup import git, path from bup.compat import bytes_from_byte, environ, range from bup.helpers import localtime, log, mkdirp, readpipe from buptest import no_lingering_errors, test_tempdir bup_exe = path.exe() def exc(*cmd): print(repr(cmd), file=sys.stderr) check_call(cmd) def exo(*cmd): print(repr(cmd), file=sys.stderr) return readpipe(cmd) @wvtest def test_git_version_detection(): with no_lingering_errors(): # Test version types from git's tag history
s.close() continue socks.append(s) if not socks: log('bup daemon: listen socket: %s\n' % e.args[1]) sys.exit(1) try: while True: [rl,wl,xl] = select.select(socks, [], [], 60) for l in rl: s, src = l.accept() try: log("Socket accepted connection from %s\n" % (src,)) fd1 = os.dup(s.fileno()) fd2 = os.dup(s.fileno()) s.close() sp = subprocess.Popen([path.exe(), 'mux', '--', path.exe(), 'server'] + extra, stdin=fd1, stdout=fd2) finally: os.close(fd1) os.close(fd2) finally: for l in socks: l.shutdown(socket.SHUT_RDWR) l.close() debug1("bup daemon: done")
p = None ret = 99 hp = extra[0].split(':') if len(hp) == 1: (hostname, port) = (hp[0], None) else: (hostname, port) = hp argv = extra[1:] p = ssh.connect(hostname, port, 'on--server', stderr=PIPE) try: argvs = '\0'.join(['bup'] + argv) p.stdin.write(struct.pack('!I', len(argvs)) + argvs) p.stdin.flush() sp = subprocess.Popen([path.exe(), 'server'], stdin=p.stdout, stdout=p.stdin) p.stdin.close() p.stdout.close() # Demultiplex remote client's stderr (back to stdout/stderr). dmc = DemuxConn(p.stderr.fileno(), open(os.devnull, "w")) for line in iter(dmc.readline, ""): sys.stdout.write(line) finally: while 1: # if we get a signal while waiting, we have to keep waiting, just # in case our child doesn't die. try: ret = p.wait() if sp: sp.wait()
socks.append(s) if not socks: log('bup daemon: listen socket: %s\n' % e.args[1]) sys.exit(1) try: while True: [rl, wl, xl] = select.select(socks, [], [], 60) for l in rl: s, src = l.accept() try: log("Socket accepted connection from %s\n" % (src, )) fd1 = os.dup(s.fileno()) fd2 = os.dup(s.fileno()) s.close() sp = subprocess.Popen( [path.exe(), 'mux', '--', path.exe(), 'server'] + extra, stdin=fd1, stdout=fd2) finally: os.close(fd1) os.close(fd2) finally: for l in socks: l.shutdown(socket.SHUT_RDWR) l.close() debug1("bup daemon: done")
from os import symlink from stat import S_IFDIR from sys import stderr from time import localtime, strftime from wvtest import * from bup import git, path, vfs from bup.compat import environ from bup.io import path_msg from bup.metadata import Metadata from bup.repo import LocalRepo, RemoteRepo from bup.test.vfs import tree_dict from buptest import ex, exo, no_lingering_errors, test_tempdir bup_path = path.exe() ## The clear_cache() calls below are to make sure that the test starts ## from a known state since at the moment the cache entry for a given ## item (like a commit) can change. For example, its meta value might ## be promoted from a mode to a Metadata instance once the tree it ## refers to is traversed. def prep_and_test_repo(name, create_repo, test_repo): with no_lingering_errors(): with test_tempdir(b'bup-t' + name) as tmpdir: bup_dir = tmpdir + b'/bup' environ[b'GIT_DIR'] = bup_dir environ[b'BUP_DIR'] = bup_dir ex((bup_path, b'init')) git.repodir = bup_dir
continue socks.append(s) if not socks: log('bup daemon: listen socket: %s\n' % e.args[1]) sys.exit(1) try: while True: [rl, wl, xl] = select.select(socks, [], [], 60) for l in rl: s, src = l.accept() try: log("Socket accepted connection from %s\n" % (src, )) fd1 = os.dup(s.fileno()) fd2 = os.dup(s.fileno()) s.close() sp = subprocess.Popen([path.exe(), 'mux', '--', 'server'] + extra, stdin=fd1, stdout=fd2) finally: os.close(fd1) os.close(fd2) finally: for l in socks: l.shutdown(socket.SHUT_RDWR) l.close() debug1("bup daemon: done")
try: hp = extra[0].split(':') if len(hp) == 1: (hostname, port) = (hp[0], None) else: (hostname, port) = hp argv = extra[1:] p = ssh.connect(hostname, port, 'on--server') argvs = '\0'.join(['bup'] + argv) p.stdin.write(struct.pack('!I', len(argvs)) + argvs) p.stdin.flush() sp = subprocess.Popen([path.exe(), 'server'], stdin=p.stdout, stdout=p.stdin) p.stdin.close() p.stdout.close() finally: while 1: # if we get a signal while waiting, we have to keep waiting, just # in case our child doesn't die. try: ret = p.wait() sp.wait() break except SigException, e: log('\nbup on: %s\n' % e)
def main(argv): o = options.Options(optspec, optfunc=getopt.getopt) opt, flags, extra = o.parse_bytes(argv[1:]) if len(extra) < 2: o.fatal('arguments expected') class SigException(Exception): def __init__(self, signum): self.signum = signum Exception.__init__(self, 'signal %d received' % signum) def handler(signum, frame): raise SigException(signum) signal.signal(signal.SIGTERM, handler) signal.signal(signal.SIGINT, handler) sys.stdout.flush() out = byte_stream(sys.stdout) try: sp = None p = None ret = 99 hp = argv_bytes(extra[0]).split(b':') if len(hp) == 1: (hostname, port) = (hp[0], None) else: (hostname, port) = hp argv = [argv_bytes(x) for x in extra[1:]] p = ssh.connect(hostname, port, b'on--server', stderr=PIPE) try: argvs = b'\0'.join([b'bup'] + argv) p.stdin.write(struct.pack('!I', len(argvs)) + argvs) p.stdin.flush() # for commands not listed here don't even execute the server # (e.g. bup on <host> index ...) cmdmodes = { b'get': b'unrestricted', b'save': b'append', b'split': b'append', b'tag': b'append', b'join': b'read', b'cat-file': b'read', b'ftp': b'read', b'ls': b'read', b'margin': b'read', b'meta': b'read', } mode = cmdmodes.get(argv[0], None) if mode is not None: # we already put BUP_DIR into the environment, which # is inherited here sp = subprocess.Popen([ path.exe(), b'server', b'--force-repo', b'--mode=' + mode ], stdin=p.stdout, stdout=p.stdin) p.stdin.close() p.stdout.close() # Demultiplex remote client's stderr (back to stdout/stderr). dmc = DemuxConn(p.stderr.fileno(), open(os.devnull, "wb")) for line in iter(dmc.readline, b''): out.write(line) finally: while 1: # if we get a signal while waiting, we have to keep waiting, just # in case our child doesn't die. try: ret = p.wait() if sp: sp.wait() break except SigException as e: log('\nbup on: %s\n' % e) os.kill(p.pid, e.signum) ret = 84 except SigException as e: if ret == 0: ret = 99 log('\nbup on: %s\n' % e) sys.exit(ret)