def finalized(enter_result=None, finalize=None):
assert finalize
try:
yield enter_result
except BaseException as ex:
with pending_raise(ex):
finalize(enter_result)
finalize(enter_result)
def __init__(self, remote, create=False):
self.closed = False
self._busy = self.conn = None
self.sock = self.p = self.pout = self.pin = None
try:
is_reverse = environ.get(b'BUP_SERVER_REVERSE')
if is_reverse:
assert(not remote)
remote = b'%s:' % is_reverse
(self.protocol, self.host, self.port, self.dir) = parse_remote(remote)
# The b'None' here matches python2's behavior of b'%s' % None == 'None',
# python3 will (as of version 3.7.5) do the same for str ('%s' % None),
# but crashes instead when doing b'%s' % None.
cachehost = b'None' if self.host is None else self.host
cachedir = b'None' if self.dir is None else self.dir
self.cachedir = git.repo(b'index-cache/%s'
% re.sub(br'[^@\w]',
b'_',
b'%s:%s' % (cachehost, cachedir)))
if is_reverse:
self.pout = os.fdopen(3, 'rb')
self.pin = os.fdopen(4, 'wb')
self.conn = Conn(self.pout, self.pin)
else:
if self.protocol in (b'ssh', b'file'):
try:
# FIXME: ssh and file shouldn't use the same module
self.p = ssh.connect(self.host, self.port, b'server')
self.pout = self.p.stdout
self.pin = self.p.stdin
self.conn = Conn(self.pout, self.pin)
except OSError as e:
reraise(ClientError('connect: %s' % e))
elif self.protocol == b'bup':
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.connect((self.host,
1982 if self.port is None else int(self.port)))
self.sockw = self.sock.makefile('wb')
self.conn = DemuxConn(self.sock.fileno(), self.sockw)
self._available_commands = self._get_available_commands()
self._require_command(b'init-dir')
self._require_command(b'set-dir')
if self.dir:
self.dir = re.sub(br'[\r\n]', ' ', self.dir)
if create:
self.conn.write(b'init-dir %s\n' % self.dir)
else:
self.conn.write(b'set-dir %s\n' % self.dir)
self.check_ok()
self.sync_indexes()
except BaseException as ex:
with pending_raise(ex):
self.close()
def test_pending_raise():
outer = Exception('outer')
inner = Exception('inner')
try:
try:
raise outer
except Exception as ex:
with pending_raise(ex):
pass
except Exception as ex:
wvpasseq(outer, ex)
wvpasseq(None, getattr(outer, '__context__', None))
try:
try:
raise outer
except Exception as ex:
with pending_raise(ex):
raise inner
except Exception as ex:
wvpasseq(inner, ex)
wvpasseq(None, getattr(outer, '__context__', None))
wvpasseq(outer, getattr(inner, '__context__', None))
def bup_rm(repo, paths, compression=6, verbosity=None):
dead_branches, dead_saves = dead_items(repo, paths)
die_if_errors('not proceeding with any removals\n')
updated_refs = {} # ref_name -> (original_ref, tip_commit(bin))
for branchname, branchitem in dead_branches.items():
ref = b'refs/heads/' + branchname
assert(not ref in updated_refs)
updated_refs[ref] = (branchitem.oid, None)
if dead_saves:
writer = git.PackWriter(compression_level=compression)
try:
for branch, saves in dead_saves.items():
assert(saves)
updated_refs[b'refs/heads/' + branch] = rm_saves(saves, writer)
except BaseException as ex:
with pending_raise(ex):
writer.abort()
finally:
writer.close()
# Only update the refs here, at the very end, so that if something
# goes wrong above, the old refs will be undisturbed. Make an attempt
# to update each ref.
for ref_name, info in updated_refs.items():
orig_ref, new_ref = info
try:
if not new_ref:
git.delete_ref(ref_name, hexlify(orig_ref))
else:
git.update_ref(ref_name, new_ref, orig_ref)
if verbosity:
log('updated %s (%s%s)\n'
% (path_msg(ref_name),
hexstr(orig_ref) + ' -> ' if orig_ref else '',
hexstr(new_ref)))
except (git.GitError, ClientError) as ex:
if new_ref:
add_error('while trying to update %s (%s%s): %s'
% (path_msg(ref_name),
hexstr(orig_ref) + ' -> ' if orig_ref else '',
hexstr(new_ref),
ex))
else:
add_error('while trying to delete %r (%s): %s'
% (ref_name, hexstr(orig_ref), ex))
def __init__(self, infd, outp):
BaseConn.__init__(self, outp)
# Anything that comes through before the sync string was not
# multiplexed and can be assumed to be debug/log before mux init.
stderr = byte_stream(sys.stderr)
cookie = b'BUPMUX'
pos = 0
while True:
b = os.read(infd, 1)
# Make sure to write all pre-BUPMUX output to stderr
if not b:
ex = IOError('demux: unexpected EOF during initialization')
with pending_raise(ex):
stderr.write(cookie[:pos])
stderr.flush()
if b == bytes_from_byte(cookie[pos]):
pos += 1
if pos == len(cookie):
break
continue
# If we can't find a new char of 'BUPMUX' then we must have some
# pre-mux log messages - output those as soon and as complete as
# possible.
#
# \r\n interacts badly with print_clean_line() in the main bup
# so remove all the \r so we see the full the lines. This assumes
# that nothing at this point will intentionally delete lines, but
# as we're just during SSH init that seems reasonable.
if b == b'\r':
continue
stderr.write(cookie[:pos]) # could be we have "BU" in the logs or so
pos = 0
stderr.write(b) # pre-mux log messages
stderr.flush()
self.infd = infd
self.reader = None
self.buf = None
self.closed = False
def __init__(self, infd, outp):
BaseConn.__init__(self, outp)
# Anything that comes through before the sync string was not
# multiplexed and can be assumed to be debug/log before mux init.
tail = b''
stderr = byte_stream(sys.stderr)
while tail != b'BUPMUX':
# Make sure to write all pre-BUPMUX output to stderr
b = os.read(infd, (len(tail) < 6) and (6 - len(tail)) or 1)
if not b:
ex = IOError('demux: unexpected EOF during initialization')
with pending_raise(ex):
stderr.write(tail)
stderr.flush()
tail += b
stderr.write(tail[:-6])
tail = tail[-6:]
stderr.flush()
self.infd = infd
self.reader = None
self.buf = None
self.closed = False
def __exit__(self, type, value, traceback):
with pending_raise(value, rethrow=False):
self.close()
def __exit__(self, exc_type, exc_value, tb):
with pending_raise(exc_value, rethrow=False):
self.close()
def sweep(live_objects, existing_count, cat_pipe, threshold, compression,
verbosity):
# Traverse all the packs, saving the (probably) live data.
ns = Nonlocal()
ns.stale_files = []
def remove_stale_files(new_pack_prefix):
if verbosity and new_pack_prefix:
log('created ' + path_msg(basename(new_pack_prefix)) + '\n')
for p in ns.stale_files:
if new_pack_prefix and p.startswith(new_pack_prefix):
continue # Don't remove the new pack file
if verbosity:
log('removing ' + path_msg(basename(p)) + '\n')
os.unlink(p)
if ns.stale_files: # So git cat-pipe will close them
cat_pipe.restart()
ns.stale_files = []
writer = git.PackWriter(objcache_maker=None,
compression_level=compression,
run_midx=False,
on_pack_finish=remove_stale_files)
try:
# FIXME: sanity check .idx names vs .pack names?
collect_count = 0
for idx_name in glob.glob(
os.path.join(git.repo(b'objects/pack'), b'*.idx')):
if verbosity:
qprogress('preserving live data (%d%% complete)\r' %
((float(collect_count) / existing_count) * 100))
with git.open_idx(idx_name) as idx:
idx_live_count = 0
for sha in idx:
if live_objects.exists(sha):
idx_live_count += 1
collect_count += idx_live_count
if idx_live_count == 0:
if verbosity:
log('deleting %s\n' %
path_msg(git.repo_rel(basename(idx_name))))
ns.stale_files.append(idx_name)
ns.stale_files.append(idx_name[:-3] + b'pack')
continue
live_frac = idx_live_count / float(len(idx))
if live_frac > ((100 - threshold) / 100.0):
if verbosity:
log('keeping %s (%d%% live)\n' % (git.repo_rel(
basename(idx_name)), live_frac * 100))
continue
if verbosity:
log('rewriting %s (%.2f%% live)\n' %
(basename(idx_name), live_frac * 100))
for sha in idx:
if live_objects.exists(sha):
item_it = cat_pipe.get(hexlify(sha))
_, typ, _ = next(item_it)
writer.just_write(sha, typ, b''.join(item_it))
ns.stale_files.append(idx_name)
ns.stale_files.append(idx_name[:-3] + b'pack')
if verbosity:
progress('preserving live data (%d%% complete)\n' %
((float(collect_count) / existing_count) * 100))
# Nothing should have recreated midx/bloom yet.
pack_dir = git.repo(b'objects/pack')
assert (not os.path.exists(os.path.join(pack_dir, b'bup.bloom')))
assert (not glob.glob(os.path.join(pack_dir, b'*.midx')))
except BaseException as ex:
with pending_raise(ex):
writer.abort()
# This will finally run midx.
# Can only change refs (if needed) after this.
writer.close()
remove_stale_files(None) # In case we didn't write to the writer.
if verbosity:
log('discarded %d%% of objects\n' %
((existing_count - count_objects(pack_dir, verbosity)) /
float(existing_count) * 100))
def receive_objects_v2(conn, junk):
global suspended_w
_init_session()
if suspended_w:
w = suspended_w
suspended_w = None
elif dumb_server_mode:
w = git.PackWriter(objcache_maker=None)
else:
w = git.PackWriter()
try:
suggested = set()
while 1:
ns = conn.read(4)
if not ns:
w.abort()
raise Exception('object read: expected length header, got EOF')
n = struct.unpack('!I', ns)[0]
#debug2('expecting %d bytes\n' % n)
if not n:
debug1('bup server: received %d object%s.\n'
% (w.count, w.count!=1 and "s" or ''))
fullpath = w.close(run_midx=not dumb_server_mode)
w = None
if fullpath:
dir, name = os.path.split(fullpath)
conn.write(b'%s.idx\n' % name)
conn.ok()
return
elif n == 0xffffffff:
debug2('bup server: receive-objects suspending\n')
conn.ok()
suspended_w = w
w = None
return
shar = conn.read(20)
crcr = struct.unpack('!I', conn.read(4))[0]
n -= 20 + 4
buf = conn.read(n) # object sizes in bup are reasonably small
#debug2('read %d bytes\n' % n)
_check(w, n, len(buf), 'object read: expected %d bytes, got %d\n')
if not dumb_server_mode:
oldpack = w.exists(shar, want_source=True)
if oldpack:
assert(not oldpack == True)
assert(oldpack.endswith(b'.idx'))
(dir,name) = os.path.split(oldpack)
if not (name in suggested):
debug1("bup server: suggesting index %s\n"
% git.shorten_hash(name).decode('ascii'))
debug1("bup server: because of object %s\n"
% hexstr(shar))
conn.write(b'index %s\n' % name)
suggested.add(name)
continue
nw, crc = w._raw_write((buf,), sha=shar)
_check(w, crcr, crc, 'object read: expected crc %d, got %d\n')
# py2: this clause is unneeded with py3
except BaseException as ex:
with pending_raise(ex):
if w:
w, w_tmp = None, w
w_tmp.close()
finally:
if w: w.close()
assert False # should be unreachable
def __exit__(self, type, value, traceback):
with pending_raise(value, rethrow=True):
self.abort_save()
