def _suggest_packs(self):
ob = self._busy
if ob:
assert(ob == 'receive-objects-v2')
self.conn.write('\xff\xff\xff\xff') # suspend receive-objects-v2
suggested = []
for line in linereader(self.conn):
if not line:
break
debug2('%s\n' % line)
if line.startswith('index '):
idx = line[6:]
debug1('client: received index suggestion: %s\n'
% git.shorten_hash(idx))
suggested.append(idx)
else:
assert(line.endswith('.idx'))
debug1('client: completed writing pack, idx: %s\n'
% git.shorten_hash(line))
suggested.append(line)
self.check_ok()
if ob:
self._busy = None
idx = None
for idx in suggested:
self.sync_index(idx)
git.auto_midx(self.cachedir)
if ob:
self._busy = ob
self.conn.write('%s\n' % ob)
return idx
def rev_parse(committish, repo_dir=None):
"""Resolve the full hash for 'committish', if it exists.
Should be roughly equivalent to 'git rev-parse'.
Returns the hex value of the hash if it is found, None if 'committish' does
not correspond to anything.
"""
head = read_ref(committish, repo_dir=repo_dir)
if head:
debug2("resolved from ref: commit = %s\n" % head.encode('hex'))
return head
pL = PackIdxList(repo('objects/pack', repo_dir=repo_dir))
if len(committish) == 40:
try:
hash = committish.decode('hex')
except TypeError:
return None
if pL.exists(hash):
return hash
return None
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if extra:
o.fatal('no arguments expected')
debug2('bup server: reading from stdin.\n')
class ServerRepo(LocalRepo):
def __init__(self, repo_dir):
if opt.force_repo:
repo_dir = None
LocalRepo.__init__(self, repo_dir)
def _restrict(server, commands):
for fn in dir(server):
if getattr(fn, 'bup_server_command', False):
if not fn in commands:
del cls.fn
modes = ['unrestricted', 'append', 'read-append', 'read']
if opt.mode is not None and opt.mode not in modes:
o.fatal("server: invalid mode")
BupProtocolServer(Conn(byte_stream(sys.stdin), byte_stream(sys.stdout)),
ServerRepo, mode=opt.mode).handle()
def _suggest_packs(self):
ob = self._busy
if ob:
assert(ob == b'receive-objects-v2')
self.conn.write(b'\xff\xff\xff\xff') # suspend receive-objects-v2
suggested = []
for line in linereader(self.conn):
if not line:
break
debug2('%r\n' % line)
if line.startswith(b'index '):
idx = line[6:]
debug1('client: received index suggestion: %s\n'
% git.shorten_hash(idx).decode('ascii'))
suggested.append(idx)
else:
assert(line.endswith(b'.idx'))
debug1('client: completed writing pack, idx: %s\n'
% git.shorten_hash(line).decode('ascii'))
suggested.append(line)
self.check_ok()
if ob:
self._busy = None
idx = None
for idx in suggested:
self.sync_index(idx)
git.auto_midx(self.cachedir)
if ob:
self._busy = ob
self.conn.write(b'%s\n' % ob)
return idx
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if extra:
o.fatal('no arguments expected')
debug2('bup server: reading from stdin.\n')
# FIXME: this protocol is totally lame and not at all future-proof.
# (Especially since we abort completely as soon as *anything* bad happens)
sys.stdout.flush()
conn = Conn(byte_stream(sys.stdin), byte_stream(sys.stdout))
lr = linereader(conn)
for _line in lr:
line = _line.strip()
if not line:
continue
debug1('bup server: command: %r\n' % line)
words = line.split(b' ', 1)
cmd = words[0]
rest = len(words)>1 and words[1] or b''
if cmd == b'quit':
break
else:
cmd = commands.get(cmd)
if cmd:
cmd(conn, rest)
else:
raise Exception('unknown server command: %r\n' % line)
debug1('bup server: done\n')
def rev_parse(committish, repo_dir=None):
"""Resolve the full hash for 'committish', if it exists.
Should be roughly equivalent to 'git rev-parse'.
Returns the hex value of the hash if it is found, None if 'committish' does
not correspond to anything.
"""
head = read_ref(committish, repo_dir=repo_dir)
if head:
debug2("resolved from ref: commit = %s\n" % hexlify(head))
return head
pL = PackIdxList(repo(b'objects/pack', repo_dir=repo_dir))
if len(committish) == 40:
try:
hash = unhexlify(committish)
except TypeError:
return None
if pL.exists(hash):
return hash
return None
def receive_objects_v2(conn, junk):
global suspended_w
_init_session()
suggested = set()
if suspended_w:
w = suspended_w
suspended_w = None
else:
if dumb_server_mode:
w = git.PackWriter(objcache_maker=None)
else:
w = git.PackWriter()
while 1:
ns = conn.read(4)
if not ns:
w.abort()
raise Exception('object read: expected length header, got EOF\n')
n = struct.unpack('!I', ns)[0]
#debug2('expecting %d bytes\n' % n)
if not n:
debug1('bup server: received %d object%s.\n'
% (w.count, w.count!=1 and "s" or ''))
fullpath = w.close(run_midx=not dumb_server_mode)
if fullpath:
(dir, name) = os.path.split(fullpath)
conn.write('%s.idx\n' % name)
conn.ok()
return
elif n == 0xffffffff:
debug2('bup server: receive-objects suspended.\n')
suspended_w = w
conn.ok()
return
shar = conn.read(20)
crcr = struct.unpack('!I', conn.read(4))[0]
n -= 20 + 4
buf = conn.read(n) # object sizes in bup are reasonably small
#debug2('read %d bytes\n' % n)
_check(w, n, len(buf), 'object read: expected %d bytes, got %d\n')
if not dumb_server_mode:
oldpack = w.exists(shar, want_source=True)
if oldpack:
assert(not oldpack == True)
assert(oldpack.endswith('.idx'))
(dir,name) = os.path.split(oldpack)
if not (name in suggested):
debug1("bup server: suggesting index %s\n"
% git.shorten_hash(name))
debug1("bup server: because of object %s\n"
% shar.encode('hex'))
conn.write('index %s\n' % name)
suggested.add(name)
continue
nw, crc = w._raw_write((buf,), sha=shar)
_check(w, crcr, crc, 'object read: expected crc %d, got %d\n')
def receive_objects_v2(conn, junk):
global suspended_w
_init_session()
suggested = set()
if suspended_w:
w = suspended_w
suspended_w = None
else:
if dumb_server_mode:
w = git.PackWriter(objcache_maker=None)
else:
w = git.PackWriter()
while 1:
ns = conn.read(4)
if not ns:
w.abort()
raise Exception('object read: expected length header, got EOF\n')
n = struct.unpack('!I', ns)[0]
#debug2('expecting %d bytes\n' % n)
if not n:
debug1('bup server: received %d object%s.\n'
% (w.count, w.count!=1 and "s" or ''))
fullpath = w.close(run_midx=not dumb_server_mode)
if fullpath:
(dir, name) = os.path.split(fullpath)
conn.write('%s.idx\n' % name)
conn.ok()
return
elif n == 0xffffffff:
debug2('bup server: receive-objects suspended.\n')
suspended_w = w
conn.ok()
return
shar = conn.read(20)
crcr = struct.unpack('!I', conn.read(4))[0]
n -= 20 + 4
buf = conn.read(n) # object sizes in bup are reasonably small
#debug2('read %d bytes\n' % n)
_check(w, n, len(buf), 'object read: expected %d bytes, got %d\n')
if not dumb_server_mode:
oldpack = w.exists(shar, want_source=True)
if oldpack:
assert(not oldpack == True)
assert(oldpack.endswith('.idx'))
(dir,name) = os.path.split(oldpack)
if not (name in suggested):
debug1("bup server: suggesting index %s\n"
% git.shorten_hash(name))
debug1("bup server: because of object %s\n"
% shar.encode('hex'))
conn.write('index %s\n' % name)
suggested.add(name)
continue
nw, crc = w._raw_write((buf,), sha=shar)
_check(w, crcr, crc, 'object read: expected crc %d, got %d\n')
def close(self):
if self.map and self.rwfile:
debug2("bloom: closing with %d entries\n" % self.entries)
self.map[12:16] = struct.pack('!I', self.entries)
if self.delaywrite:
self.rwfile.seek(0)
self.rwfile.write(self.map)
else:
self.map.flush()
self.rwfile.seek(16 + 2**self.bits)
if self.idxnames:
self.rwfile.write('\0'.join(self.idxnames))
self._init_failed()
def close(self):
if self.map and self.rwfile:
debug2("bloom: closing with %d entries\n" % self.entries)
self.map[12:16] = struct.pack('!I', self.entries)
if self.delaywrite:
self.rwfile.seek(0)
self.rwfile.write(self.map)
else:
self.map.flush()
self.rwfile.seek(16 + 2**self.bits)
if self.idxnames:
self.rwfile.write('\0'.join(self.idxnames))
self._init_failed()
def close(self):
self.closed = True
try:
if self.map and self.readwrite:
debug2("bloom: closing with %d entries\n" % self.entries)
self.map[12:16] = struct.pack('!I', self.entries)
if self.delaywrite:
self.file.seek(0)
self.file.write(self.map)
else:
self.map.flush()
self.file.seek(16 + 2**self.bits)
if self.idxnames:
self.file.write(b'\0'.join(self.idxnames))
finally: # This won't handle pending exceptions correctly in py2
self._init_failed()
def next(self, size):
out = ''
while len(out) < size:
if self.it and not self.blob:
try:
self.blob = self.it.next()
except StopIteration:
self.it = None
if self.blob:
want = size - len(out)
out += self.blob[:want]
self.blob = self.blob[want:]
if not self.it:
break
debug2('next(%d) returned %d\n' % (size, len(out)))
self.ofs += len(out)
return out
def main(argv):
# Give the subcommand exclusive access to stdin.
orig_stdin = os.dup(0)
devnull = os.open(os.devnull, os.O_RDONLY)
os.dup2(devnull, 0)
os.close(devnull)
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if len(extra) < 1:
o.fatal('command is required')
subcmd = extra
debug2('bup mux: starting %r\n' % (extra, ))
outr, outw = os.pipe()
errr, errw = os.pipe()
def close_fds():
os.close(outr)
os.close(errr)
p = subprocess.Popen(subcmd,
stdin=orig_stdin,
stdout=outw,
stderr=errw,
close_fds=False,
preexec_fn=close_fds)
os.close(outw)
os.close(errw)
sys.stdout.flush()
out = byte_stream(sys.stdout)
out.write(b'BUPMUX')
out.flush()
mux(p, out.fileno(), outr, errr)
os.close(outr)
os.close(errr)
prv = p.wait()
if prv:
debug1('%s exited with code %d\n' % (extra[0], prv))
debug1('bup mux: done\n')
sys.exit(prv)
def next(self, size):
out = b''
while len(out) < size:
if self.it and not self.blob:
try:
self.blob = next(self.it)
except StopIteration:
self.it = None
if self.blob:
want = size - len(out)
out += self.blob[:want]
self.blob = self.blob[want:]
if not self.it:
break
debug2('next(%d) returned %d\n' % (size, len(out)))
self.ofs += len(out)
return out
def rev_parse(self, committish):
"""Resolve the full hash for 'committish', if it exists.
Should be roughly equivalent to 'git rev-parse'.
Returns the hex value of the hash if it is found, None if 'committish' does
not correspond to anything.
"""
head = self.read_ref(committish)
if head:
debug2("resolved from ref: commit = %s\n" % hexlify(head))
return head
if len(committish) == 40:
try:
hash = unhexlify(committish)
except TypeError:
return None
if self.exists(hash):
return hash
return None
devnull = os.open('/dev/null', os.O_RDONLY)
os.dup2(devnull, 0)
os.close(devnull)
optspec = """
bup mux command [arguments...]
--
"""
o = options.Options(optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])
if len(extra) < 1:
o.fatal('command is required')
subcmd = extra
debug2('bup mux: starting %r\n' % (extra,))
outr, outw = os.pipe()
errr, errw = os.pipe()
def close_fds():
os.close(outr)
os.close(errr)
p = subprocess.Popen(subcmd, stdin=orig_stdin, stdout=outw, stderr=errw,
preexec_fn=close_fds)
os.close(outw)
os.close(errw)
sys.stdout.write('BUPMUX')
sys.stdout.flush()
mux(p, sys.stdout.fileno(), outr, errr)
os.close(outr)
msg = 'git rev-list returned error %d' % rv
conn.error(msg)
raise GitError(msg)
conn.ok()
optspec = """
bup server
"""
o = options.Options(optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])
if extra:
o.fatal('no arguments expected')
debug2('bup server: reading from stdin.\n')
commands = {
'quit': None,
'help': do_help,
'init-dir': init_dir,
'set-dir': set_dir,
'list-indexes': list_indexes,
'send-index': send_index,
'receive-objects-v2': receive_objects_v2,
'read-ref': read_ref,
'update-ref': update_ref,
'join': join,
'cat': join, # apocryphal alias
'cat-batch' : cat_batch,
'refs': refs,
conn.error(e)
else:
conn.write("\0\0\0\0")
conn.ok()
optspec = """
bup server
"""
o = options.Options(optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])
if extra:
o.fatal("no arguments expected")
debug2("bup server: reading from stdin.\n")
commands = {
"quit": None,
"help": do_help,
"init-dir": init_dir,
"set-dir": set_dir,
"list-indexes": list_indexes,
"send-index": send_index,
"receive-objects-v2": receive_objects_v2,
"read-ref": read_ref,
"update-ref": update_ref,
"cat": cat,
}
# FIXME: this protocol is totally lame and not at all future-proof.
devnull = os.open(os.devnull, os.O_RDONLY)
os.dup2(devnull, 0)
os.close(devnull)
optspec = """
bup mux command [arguments...]
--
"""
o = options.Options(optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])
if len(extra) < 1:
o.fatal('command is required')
subcmd = extra
debug2('bup mux: starting %r\n' % (extra,))
outr, outw = os.pipe()
errr, errw = os.pipe()
def close_fds():
os.close(outr)
os.close(errr)
p = subprocess.Popen(subcmd, stdin=orig_stdin, stdout=outw, stderr=errw,
close_fds=False, preexec_fn=close_fds)
os.close(outw)
os.close(errw)
sys.stdout.flush()
out = byte_stream(sys.stdout)
out.write(b'BUPMUX')
out.flush()
msg = 'git rev-list returned error %d' % rv
conn.error(msg)
raise GitError(msg)
conn.ok()
optspec = """
bup server
"""
o = options.Options(optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])
if extra:
o.fatal('no arguments expected')
debug2('bup server: reading from stdin.\n')
commands = {
'quit': None,
'help': do_help,
'init-dir': init_dir,
'set-dir': set_dir,
'list-indexes': list_indexes,
'send-index': send_index,
'receive-objects-v2': receive_objects_v2,
'read-ref': read_ref,
'update-ref': update_ref,
'join': join,
'cat': join, # apocryphal alias
'cat-batch' : cat_batch,
'refs': refs,
def receive_objects_v2(self, junk):
self.init_session()
suggested = set()
if self.suspended:
self.suspended = False
else:
if self.dumb_server_mode:
objcache_maker = lambda: None
else:
objcache_maker = None
# FIXME: this goes together with the direct accesses below
self.repo._ensure_packwriter()
while 1:
ns = self.conn.read(4)
if not ns:
self.repo.abort_writing()
raise Exception(
'object read: expected length header, got EOF\n')
n = struct.unpack('!I', ns)[0]
#debug2('expecting %d bytes\n' % n)
if not n:
# FIXME: don't be lazy and count ourselves, or something, at least
# don't access self.repo internals
debug1('bup server: received %d object%s.\n' %
(self.repo._packwriter.count,
self.repo._packwriter.count != 1 and "s" or ''))
fullpath = self.repo.finish_writing(
run_midx=not self.dumb_server_mode)
if fullpath:
(dir, name) = os.path.split(fullpath)
self.conn.write(b'%s.idx\n' % name)
self.conn.ok()
return
elif n == 0xffffffff:
debug2('bup server: receive-objects suspended.\n')
self.suspended = True
self.conn.ok()
return
shar = self.conn.read(20)
crcr = struct.unpack('!I', self.conn.read(4))[0]
n -= 20 + 4
buf = self.conn.read(n) # object sizes in bup are reasonably small
#debug2('read %d bytes\n' % n)
self._check(n, len(buf),
'object read: expected %d bytes, got %d\n')
if not self.dumb_server_mode:
result = self.repo.exists(shar, want_source=True)
if result:
oldpack = result.pack
assert (oldpack.endswith(b'.idx'))
(dir, name) = os.path.split(oldpack)
if not (name in suggested):
debug1("bup server: suggesting index %s\n" %
git.shorten_hash(name).decode('ascii'))
debug1("bup server: because of object %s\n" %
hexstr(shar))
self.conn.write(b'index %s\n' % name)
suggested.add(name)
continue
# FIXME: figure out the right abstraction for this; or better yet,
# make the protocol aware of the object type
nw, crc = self.repo._packwriter._raw_write((buf, ), sha=shar)
self._check(crcr, crc, 'object read: expected crc %d, got %d\n')