Exemplo n.º 1
0
Arquivo: client.py Projeto: xx4h/bup
 def _suggest_packs(self):
     ob = self._busy
     if ob:
         assert(ob == 'receive-objects-v2')
         self.conn.write('\xff\xff\xff\xff')  # suspend receive-objects-v2
     suggested = []
     for line in linereader(self.conn):
         if not line:
             break
         debug2('%s\n' % line)
         if line.startswith('index '):
             idx = line[6:]
             debug1('client: received index suggestion: %s\n'
                    % git.shorten_hash(idx))
             suggested.append(idx)
         else:
             assert(line.endswith('.idx'))
             debug1('client: completed writing pack, idx: %s\n'
                    % git.shorten_hash(line))
             suggested.append(line)
     self.check_ok()
     if ob:
         self._busy = None
     idx = None
     for idx in suggested:
         self.sync_index(idx)
     git.auto_midx(self.cachedir)
     if ob:
         self._busy = ob
         self.conn.write('%s\n' % ob)
     return idx
Exemplo n.º 2
0
def rev_parse(committish, repo_dir=None):
    """Resolve the full hash for 'committish', if it exists.

    Should be roughly equivalent to 'git rev-parse'.

    Returns the hex value of the hash if it is found, None if 'committish' does
    not correspond to anything.
    """
    head = read_ref(committish, repo_dir=repo_dir)
    if head:
        debug2("resolved from ref: commit = %s\n" % head.encode('hex'))
        return head

    pL = PackIdxList(repo('objects/pack', repo_dir=repo_dir))

    if len(committish) == 40:
        try:
            hash = committish.decode('hex')
        except TypeError:
            return None

        if pL.exists(hash):
            return hash

    return None
Exemplo n.º 3
0
Arquivo: server.py Projeto: jmberg/bup
def main(argv):
    o = options.Options(optspec)
    opt, flags, extra = o.parse_bytes(argv[1:])

    if extra:
        o.fatal('no arguments expected')

    debug2('bup server: reading from stdin.\n')

    class ServerRepo(LocalRepo):
        def __init__(self, repo_dir):
            if opt.force_repo:
                repo_dir = None
            LocalRepo.__init__(self, repo_dir)

    def _restrict(server, commands):
        for fn in dir(server):
            if getattr(fn, 'bup_server_command', False):
                if not fn in commands:
                    del cls.fn

    modes = ['unrestricted', 'append', 'read-append', 'read']
    if opt.mode is not None and opt.mode not in modes:
        o.fatal("server: invalid mode")

    BupProtocolServer(Conn(byte_stream(sys.stdin), byte_stream(sys.stdout)),
                      ServerRepo, mode=opt.mode).handle()
Exemplo n.º 4
0
 def _suggest_packs(self):
     ob = self._busy
     if ob:
         assert(ob == b'receive-objects-v2')
         self.conn.write(b'\xff\xff\xff\xff')  # suspend receive-objects-v2
     suggested = []
     for line in linereader(self.conn):
         if not line:
             break
         debug2('%r\n' % line)
         if line.startswith(b'index '):
             idx = line[6:]
             debug1('client: received index suggestion: %s\n'
                    % git.shorten_hash(idx).decode('ascii'))
             suggested.append(idx)
         else:
             assert(line.endswith(b'.idx'))
             debug1('client: completed writing pack, idx: %s\n'
                    % git.shorten_hash(line).decode('ascii'))
             suggested.append(line)
     self.check_ok()
     if ob:
         self._busy = None
     idx = None
     for idx in suggested:
         self.sync_index(idx)
     git.auto_midx(self.cachedir)
     if ob:
         self._busy = ob
         self.conn.write(b'%s\n' % ob)
     return idx
Exemplo n.º 5
0
def main(argv):
    o = options.Options(optspec)
    opt, flags, extra = o.parse_bytes(argv[1:])

    if extra:
        o.fatal('no arguments expected')

    debug2('bup server: reading from stdin.\n')

    # FIXME: this protocol is totally lame and not at all future-proof.
    # (Especially since we abort completely as soon as *anything* bad happens)
    sys.stdout.flush()
    conn = Conn(byte_stream(sys.stdin), byte_stream(sys.stdout))
    lr = linereader(conn)
    for _line in lr:
        line = _line.strip()
        if not line:
            continue
        debug1('bup server: command: %r\n' % line)
        words = line.split(b' ', 1)
        cmd = words[0]
        rest = len(words)>1 and words[1] or b''
        if cmd == b'quit':
            break
        else:
            cmd = commands.get(cmd)
            if cmd:
                cmd(conn, rest)
            else:
                raise Exception('unknown server command: %r\n' % line)

    debug1('bup server: done\n')
Exemplo n.º 6
0
Arquivo: git.py Projeto: zzmjohn/bup
def rev_parse(committish, repo_dir=None):
    """Resolve the full hash for 'committish', if it exists.

    Should be roughly equivalent to 'git rev-parse'.

    Returns the hex value of the hash if it is found, None if 'committish' does
    not correspond to anything.
    """
    head = read_ref(committish, repo_dir=repo_dir)
    if head:
        debug2("resolved from ref: commit = %s\n" % hexlify(head))
        return head

    pL = PackIdxList(repo(b'objects/pack', repo_dir=repo_dir))

    if len(committish) == 40:
        try:
            hash = unhexlify(committish)
        except TypeError:
            return None

        if pL.exists(hash):
            return hash

    return None
Exemplo n.º 7
0
def receive_objects_v2(conn, junk):
    global suspended_w
    _init_session()
    suggested = set()
    if suspended_w:
        w = suspended_w
        suspended_w = None
    else:
        if dumb_server_mode:
            w = git.PackWriter(objcache_maker=None)
        else:
            w = git.PackWriter()
    while 1:
        ns = conn.read(4)
        if not ns:
            w.abort()
            raise Exception('object read: expected length header, got EOF\n')
        n = struct.unpack('!I', ns)[0]
        #debug2('expecting %d bytes\n' % n)
        if not n:
            debug1('bup server: received %d object%s.\n' 
                % (w.count, w.count!=1 and "s" or ''))
            fullpath = w.close(run_midx=not dumb_server_mode)
            if fullpath:
                (dir, name) = os.path.split(fullpath)
                conn.write('%s.idx\n' % name)
            conn.ok()
            return
        elif n == 0xffffffff:
            debug2('bup server: receive-objects suspended.\n')
            suspended_w = w
            conn.ok()
            return
            
        shar = conn.read(20)
        crcr = struct.unpack('!I', conn.read(4))[0]
        n -= 20 + 4
        buf = conn.read(n)  # object sizes in bup are reasonably small
        #debug2('read %d bytes\n' % n)
        _check(w, n, len(buf), 'object read: expected %d bytes, got %d\n')
        if not dumb_server_mode:
            oldpack = w.exists(shar, want_source=True)
            if oldpack:
                assert(not oldpack == True)
                assert(oldpack.endswith('.idx'))
                (dir,name) = os.path.split(oldpack)
                if not (name in suggested):
                    debug1("bup server: suggesting index %s\n"
                           % git.shorten_hash(name))
                    debug1("bup server:   because of object %s\n"
                           % shar.encode('hex'))
                    conn.write('index %s\n' % name)
                    suggested.add(name)
                continue
        nw, crc = w._raw_write((buf,), sha=shar)
        _check(w, crcr, crc, 'object read: expected crc %d, got %d\n')
Exemplo n.º 8
0
def receive_objects_v2(conn, junk):
    global suspended_w
    _init_session()
    suggested = set()
    if suspended_w:
        w = suspended_w
        suspended_w = None
    else:
        if dumb_server_mode:
            w = git.PackWriter(objcache_maker=None)
        else:
            w = git.PackWriter()
    while 1:
        ns = conn.read(4)
        if not ns:
            w.abort()
            raise Exception('object read: expected length header, got EOF\n')
        n = struct.unpack('!I', ns)[0]
        #debug2('expecting %d bytes\n' % n)
        if not n:
            debug1('bup server: received %d object%s.\n' 
                % (w.count, w.count!=1 and "s" or ''))
            fullpath = w.close(run_midx=not dumb_server_mode)
            if fullpath:
                (dir, name) = os.path.split(fullpath)
                conn.write('%s.idx\n' % name)
            conn.ok()
            return
        elif n == 0xffffffff:
            debug2('bup server: receive-objects suspended.\n')
            suspended_w = w
            conn.ok()
            return
            
        shar = conn.read(20)
        crcr = struct.unpack('!I', conn.read(4))[0]
        n -= 20 + 4
        buf = conn.read(n)  # object sizes in bup are reasonably small
        #debug2('read %d bytes\n' % n)
        _check(w, n, len(buf), 'object read: expected %d bytes, got %d\n')
        if not dumb_server_mode:
            oldpack = w.exists(shar, want_source=True)
            if oldpack:
                assert(not oldpack == True)
                assert(oldpack.endswith('.idx'))
                (dir,name) = os.path.split(oldpack)
                if not (name in suggested):
                    debug1("bup server: suggesting index %s\n"
                           % git.shorten_hash(name))
                    debug1("bup server:   because of object %s\n"
                           % shar.encode('hex'))
                    conn.write('index %s\n' % name)
                    suggested.add(name)
                continue
        nw, crc = w._raw_write((buf,), sha=shar)
        _check(w, crcr, crc, 'object read: expected crc %d, got %d\n')
Exemplo n.º 9
0
Arquivo: bloom.py Projeto: bup/bup
 def close(self):
     if self.map and self.rwfile:
         debug2("bloom: closing with %d entries\n" % self.entries)
         self.map[12:16] = struct.pack('!I', self.entries)
         if self.delaywrite:
             self.rwfile.seek(0)
             self.rwfile.write(self.map)
         else:
             self.map.flush()
         self.rwfile.seek(16 + 2**self.bits)
         if self.idxnames:
             self.rwfile.write('\0'.join(self.idxnames))
     self._init_failed()
Exemplo n.º 10
0
Arquivo: bloom.py Projeto: yafey/bup
 def close(self):
     if self.map and self.rwfile:
         debug2("bloom: closing with %d entries\n" % self.entries)
         self.map[12:16] = struct.pack('!I', self.entries)
         if self.delaywrite:
             self.rwfile.seek(0)
             self.rwfile.write(self.map)
         else:
             self.map.flush()
         self.rwfile.seek(16 + 2**self.bits)
         if self.idxnames:
             self.rwfile.write('\0'.join(self.idxnames))
     self._init_failed()
Exemplo n.º 11
0
 def close(self):
     self.closed = True
     try:
         if self.map and self.readwrite:
             debug2("bloom: closing with %d entries\n" % self.entries)
             self.map[12:16] = struct.pack('!I', self.entries)
             if self.delaywrite:
                 self.file.seek(0)
                 self.file.write(self.map)
             else:
                 self.map.flush()
             self.file.seek(16 + 2**self.bits)
             if self.idxnames:
                 self.file.write(b'\0'.join(self.idxnames))
     finally:  # This won't handle pending exceptions correctly in py2
         self._init_failed()
Exemplo n.º 12
0
Arquivo: vfs.py Projeto: bup/bup
 def next(self, size):
     out = ''
     while len(out) < size:
         if self.it and not self.blob:
             try:
                 self.blob = self.it.next()
             except StopIteration:
                 self.it = None
         if self.blob:
             want = size - len(out)
             out += self.blob[:want]
             self.blob = self.blob[want:]
         if not self.it:
             break
     debug2('next(%d) returned %d\n' % (size, len(out)))
     self.ofs += len(out)
     return out
Exemplo n.º 13
0
def main(argv):
    # Give the subcommand exclusive access to stdin.
    orig_stdin = os.dup(0)
    devnull = os.open(os.devnull, os.O_RDONLY)
    os.dup2(devnull, 0)
    os.close(devnull)

    o = options.Options(optspec)
    opt, flags, extra = o.parse_bytes(argv[1:])
    if len(extra) < 1:
        o.fatal('command is required')

    subcmd = extra

    debug2('bup mux: starting %r\n' % (extra, ))

    outr, outw = os.pipe()
    errr, errw = os.pipe()

    def close_fds():
        os.close(outr)
        os.close(errr)

    p = subprocess.Popen(subcmd,
                         stdin=orig_stdin,
                         stdout=outw,
                         stderr=errw,
                         close_fds=False,
                         preexec_fn=close_fds)
    os.close(outw)
    os.close(errw)
    sys.stdout.flush()
    out = byte_stream(sys.stdout)
    out.write(b'BUPMUX')
    out.flush()
    mux(p, out.fileno(), outr, errr)
    os.close(outr)
    os.close(errr)
    prv = p.wait()

    if prv:
        debug1('%s exited with code %d\n' % (extra[0], prv))

    debug1('bup mux: done\n')

    sys.exit(prv)
Exemplo n.º 14
0
 def next(self, size):
     out = b''
     while len(out) < size:
         if self.it and not self.blob:
             try:
                 self.blob = next(self.it)
             except StopIteration:
                 self.it = None
         if self.blob:
             want = size - len(out)
             out += self.blob[:want]
             self.blob = self.blob[want:]
         if not self.it:
             break
     debug2('next(%d) returned %d\n' % (size, len(out)))
     self.ofs += len(out)
     return out
Exemplo n.º 15
0
    def rev_parse(self, committish):
        """Resolve the full hash for 'committish', if it exists.

        Should be roughly equivalent to 'git rev-parse'.

        Returns the hex value of the hash if it is found, None if 'committish' does
        not correspond to anything.
        """
        head = self.read_ref(committish)
        if head:
            debug2("resolved from ref: commit = %s\n" % hexlify(head))
            return head

        if len(committish) == 40:
            try:
                hash = unhexlify(committish)
            except TypeError:
                return None

            if self.exists(hash):
                return hash

        return None
Exemplo n.º 16
0
Arquivo: mux-cmd.py Projeto: 0xkag/bup
devnull = os.open('/dev/null', os.O_RDONLY)
os.dup2(devnull, 0)
os.close(devnull)

optspec = """
bup mux command [arguments...]
--
"""
o = options.Options(optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])
if len(extra) < 1:
    o.fatal('command is required')

subcmd = extra

debug2('bup mux: starting %r\n' % (extra,))

outr, outw = os.pipe()
errr, errw = os.pipe()
def close_fds():
    os.close(outr)
    os.close(errr)

p = subprocess.Popen(subcmd, stdin=orig_stdin, stdout=outw, stderr=errw,
                     preexec_fn=close_fds)
os.close(outw)
os.close(errw)
sys.stdout.write('BUPMUX')
sys.stdout.flush()
mux(p, sys.stdout.fileno(), outr, errr)
os.close(outr)
Exemplo n.º 17
0
        msg = 'git rev-list returned error %d' % rv
        conn.error(msg)
        raise GitError(msg)
    conn.ok()


optspec = """
bup server
"""
o = options.Options(optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])

if extra:
    o.fatal('no arguments expected')

debug2('bup server: reading from stdin.\n')

commands = {
    'quit': None,
    'help': do_help,
    'init-dir': init_dir,
    'set-dir': set_dir,
    'list-indexes': list_indexes,
    'send-index': send_index,
    'receive-objects-v2': receive_objects_v2,
    'read-ref': read_ref,
    'update-ref': update_ref,
    'join': join,
    'cat': join,  # apocryphal alias
    'cat-batch' : cat_batch,
    'refs': refs,
Exemplo n.º 18
0
        conn.error(e)
    else:
        conn.write("\0\0\0\0")
        conn.ok()


optspec = """
bup server
"""
o = options.Options(optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])

if extra:
    o.fatal("no arguments expected")

debug2("bup server: reading from stdin.\n")

commands = {
    "quit": None,
    "help": do_help,
    "init-dir": init_dir,
    "set-dir": set_dir,
    "list-indexes": list_indexes,
    "send-index": send_index,
    "receive-objects-v2": receive_objects_v2,
    "read-ref": read_ref,
    "update-ref": update_ref,
    "cat": cat,
}

# FIXME: this protocol is totally lame and not at all future-proof.
Exemplo n.º 19
0
devnull = os.open(os.devnull, os.O_RDONLY)
os.dup2(devnull, 0)
os.close(devnull)

optspec = """
bup mux command [arguments...]
--
"""
o = options.Options(optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])
if len(extra) < 1:
    o.fatal('command is required')

subcmd = extra

debug2('bup mux: starting %r\n' % (extra,))

outr, outw = os.pipe()
errr, errw = os.pipe()
def close_fds():
    os.close(outr)
    os.close(errr)

p = subprocess.Popen(subcmd, stdin=orig_stdin, stdout=outw, stderr=errw,
                     close_fds=False, preexec_fn=close_fds)
os.close(outw)
os.close(errw)
sys.stdout.flush()
out = byte_stream(sys.stdout)
out.write(b'BUPMUX')
out.flush()
Exemplo n.º 20
0
        msg = 'git rev-list returned error %d' % rv
        conn.error(msg)
        raise GitError(msg)
    conn.ok()


optspec = """
bup server
"""
o = options.Options(optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])

if extra:
    o.fatal('no arguments expected')

debug2('bup server: reading from stdin.\n')

commands = {
    'quit': None,
    'help': do_help,
    'init-dir': init_dir,
    'set-dir': set_dir,
    'list-indexes': list_indexes,
    'send-index': send_index,
    'receive-objects-v2': receive_objects_v2,
    'read-ref': read_ref,
    'update-ref': update_ref,
    'join': join,
    'cat': join,  # apocryphal alias
    'cat-batch' : cat_batch,
    'refs': refs,
Exemplo n.º 21
0
    def receive_objects_v2(self, junk):
        self.init_session()
        suggested = set()
        if self.suspended:
            self.suspended = False
        else:
            if self.dumb_server_mode:
                objcache_maker = lambda: None
            else:
                objcache_maker = None
        # FIXME: this goes together with the direct accesses below
        self.repo._ensure_packwriter()
        while 1:
            ns = self.conn.read(4)
            if not ns:
                self.repo.abort_writing()
                raise Exception(
                    'object read: expected length header, got EOF\n')
            n = struct.unpack('!I', ns)[0]
            #debug2('expecting %d bytes\n' % n)
            if not n:
                # FIXME: don't be lazy and count ourselves, or something, at least
                # don't access self.repo internals
                debug1('bup server: received %d object%s.\n' %
                       (self.repo._packwriter.count,
                        self.repo._packwriter.count != 1 and "s" or ''))
                fullpath = self.repo.finish_writing(
                    run_midx=not self.dumb_server_mode)
                if fullpath:
                    (dir, name) = os.path.split(fullpath)
                    self.conn.write(b'%s.idx\n' % name)
                self.conn.ok()
                return
            elif n == 0xffffffff:
                debug2('bup server: receive-objects suspended.\n')
                self.suspended = True
                self.conn.ok()
                return

            shar = self.conn.read(20)
            crcr = struct.unpack('!I', self.conn.read(4))[0]
            n -= 20 + 4
            buf = self.conn.read(n)  # object sizes in bup are reasonably small
            #debug2('read %d bytes\n' % n)
            self._check(n, len(buf),
                        'object read: expected %d bytes, got %d\n')
            if not self.dumb_server_mode:
                result = self.repo.exists(shar, want_source=True)
                if result:
                    oldpack = result.pack
                    assert (oldpack.endswith(b'.idx'))
                    (dir, name) = os.path.split(oldpack)
                    if not (name in suggested):
                        debug1("bup server: suggesting index %s\n" %
                               git.shorten_hash(name).decode('ascii'))
                        debug1("bup server:   because of object %s\n" %
                               hexstr(shar))
                        self.conn.write(b'index %s\n' % name)
                        suggested.add(name)
                    continue
            # FIXME: figure out the right abstraction for this; or better yet,
            #        make the protocol aware of the object type
            nw, crc = self.repo._packwriter._raw_write((buf, ), sha=shar)
            self._check(crcr, crc, 'object read: expected crc %d, got %d\n')