def finalized(enter_result=None, finalize=None): assert finalize try: yield enter_result except BaseException as ex: with pending_raise(ex): finalize(enter_result) finalize(enter_result)
def __init__(self, remote, create=False): self.closed = False self._busy = self.conn = None self.sock = self.p = self.pout = self.pin = None try: is_reverse = environ.get(b'BUP_SERVER_REVERSE') if is_reverse: assert(not remote) remote = b'%s:' % is_reverse (self.protocol, self.host, self.port, self.dir) = parse_remote(remote) # The b'None' here matches python2's behavior of b'%s' % None == 'None', # python3 will (as of version 3.7.5) do the same for str ('%s' % None), # but crashes instead when doing b'%s' % None. cachehost = b'None' if self.host is None else self.host cachedir = b'None' if self.dir is None else self.dir self.cachedir = git.repo(b'index-cache/%s' % re.sub(br'[^@\w]', b'_', b'%s:%s' % (cachehost, cachedir))) if is_reverse: self.pout = os.fdopen(3, 'rb') self.pin = os.fdopen(4, 'wb') self.conn = Conn(self.pout, self.pin) else: if self.protocol in (b'ssh', b'file'): try: # FIXME: ssh and file shouldn't use the same module self.p = ssh.connect(self.host, self.port, b'server') self.pout = self.p.stdout self.pin = self.p.stdin self.conn = Conn(self.pout, self.pin) except OSError as e: reraise(ClientError('connect: %s' % e)) elif self.protocol == b'bup': self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.sock.connect((self.host, 1982 if self.port is None else int(self.port))) self.sockw = self.sock.makefile('wb') self.conn = DemuxConn(self.sock.fileno(), self.sockw) self._available_commands = self._get_available_commands() self._require_command(b'init-dir') self._require_command(b'set-dir') if self.dir: self.dir = re.sub(br'[\r\n]', ' ', self.dir) if create: self.conn.write(b'init-dir %s\n' % self.dir) else: self.conn.write(b'set-dir %s\n' % self.dir) self.check_ok() self.sync_indexes() except BaseException as ex: with pending_raise(ex): self.close()
def test_pending_raise(): outer = Exception('outer') inner = Exception('inner') try: try: raise outer except Exception as ex: with pending_raise(ex): pass except Exception as ex: wvpasseq(outer, ex) wvpasseq(None, getattr(outer, '__context__', None)) try: try: raise outer except Exception as ex: with pending_raise(ex): raise inner except Exception as ex: wvpasseq(inner, ex) wvpasseq(None, getattr(outer, '__context__', None)) wvpasseq(outer, getattr(inner, '__context__', None))
def bup_rm(repo, paths, compression=6, verbosity=None): dead_branches, dead_saves = dead_items(repo, paths) die_if_errors('not proceeding with any removals\n') updated_refs = {} # ref_name -> (original_ref, tip_commit(bin)) for branchname, branchitem in dead_branches.items(): ref = b'refs/heads/' + branchname assert(not ref in updated_refs) updated_refs[ref] = (branchitem.oid, None) if dead_saves: writer = git.PackWriter(compression_level=compression) try: for branch, saves in dead_saves.items(): assert(saves) updated_refs[b'refs/heads/' + branch] = rm_saves(saves, writer) except BaseException as ex: with pending_raise(ex): writer.abort() finally: writer.close() # Only update the refs here, at the very end, so that if something # goes wrong above, the old refs will be undisturbed. Make an attempt # to update each ref. for ref_name, info in updated_refs.items(): orig_ref, new_ref = info try: if not new_ref: git.delete_ref(ref_name, hexlify(orig_ref)) else: git.update_ref(ref_name, new_ref, orig_ref) if verbosity: log('updated %s (%s%s)\n' % (path_msg(ref_name), hexstr(orig_ref) + ' -> ' if orig_ref else '', hexstr(new_ref))) except (git.GitError, ClientError) as ex: if new_ref: add_error('while trying to update %s (%s%s): %s' % (path_msg(ref_name), hexstr(orig_ref) + ' -> ' if orig_ref else '', hexstr(new_ref), ex)) else: add_error('while trying to delete %r (%s): %s' % (ref_name, hexstr(orig_ref), ex))
def __init__(self, infd, outp): BaseConn.__init__(self, outp) # Anything that comes through before the sync string was not # multiplexed and can be assumed to be debug/log before mux init. stderr = byte_stream(sys.stderr) cookie = b'BUPMUX' pos = 0 while True: b = os.read(infd, 1) # Make sure to write all pre-BUPMUX output to stderr if not b: ex = IOError('demux: unexpected EOF during initialization') with pending_raise(ex): stderr.write(cookie[:pos]) stderr.flush() if b == bytes_from_byte(cookie[pos]): pos += 1 if pos == len(cookie): break continue # If we can't find a new char of 'BUPMUX' then we must have some # pre-mux log messages - output those as soon and as complete as # possible. # # \r\n interacts badly with print_clean_line() in the main bup # so remove all the \r so we see the full the lines. This assumes # that nothing at this point will intentionally delete lines, but # as we're just during SSH init that seems reasonable. if b == b'\r': continue stderr.write(cookie[:pos]) # could be we have "BU" in the logs or so pos = 0 stderr.write(b) # pre-mux log messages stderr.flush() self.infd = infd self.reader = None self.buf = None self.closed = False
def __init__(self, infd, outp): BaseConn.__init__(self, outp) # Anything that comes through before the sync string was not # multiplexed and can be assumed to be debug/log before mux init. tail = b'' stderr = byte_stream(sys.stderr) while tail != b'BUPMUX': # Make sure to write all pre-BUPMUX output to stderr b = os.read(infd, (len(tail) < 6) and (6 - len(tail)) or 1) if not b: ex = IOError('demux: unexpected EOF during initialization') with pending_raise(ex): stderr.write(tail) stderr.flush() tail += b stderr.write(tail[:-6]) tail = tail[-6:] stderr.flush() self.infd = infd self.reader = None self.buf = None self.closed = False
def __exit__(self, type, value, traceback): with pending_raise(value, rethrow=False): self.close()
def __exit__(self, exc_type, exc_value, tb): with pending_raise(exc_value, rethrow=False): self.close()
def sweep(live_objects, existing_count, cat_pipe, threshold, compression, verbosity): # Traverse all the packs, saving the (probably) live data. ns = Nonlocal() ns.stale_files = [] def remove_stale_files(new_pack_prefix): if verbosity and new_pack_prefix: log('created ' + path_msg(basename(new_pack_prefix)) + '\n') for p in ns.stale_files: if new_pack_prefix and p.startswith(new_pack_prefix): continue # Don't remove the new pack file if verbosity: log('removing ' + path_msg(basename(p)) + '\n') os.unlink(p) if ns.stale_files: # So git cat-pipe will close them cat_pipe.restart() ns.stale_files = [] writer = git.PackWriter(objcache_maker=None, compression_level=compression, run_midx=False, on_pack_finish=remove_stale_files) try: # FIXME: sanity check .idx names vs .pack names? collect_count = 0 for idx_name in glob.glob( os.path.join(git.repo(b'objects/pack'), b'*.idx')): if verbosity: qprogress('preserving live data (%d%% complete)\r' % ((float(collect_count) / existing_count) * 100)) with git.open_idx(idx_name) as idx: idx_live_count = 0 for sha in idx: if live_objects.exists(sha): idx_live_count += 1 collect_count += idx_live_count if idx_live_count == 0: if verbosity: log('deleting %s\n' % path_msg(git.repo_rel(basename(idx_name)))) ns.stale_files.append(idx_name) ns.stale_files.append(idx_name[:-3] + b'pack') continue live_frac = idx_live_count / float(len(idx)) if live_frac > ((100 - threshold) / 100.0): if verbosity: log('keeping %s (%d%% live)\n' % (git.repo_rel( basename(idx_name)), live_frac * 100)) continue if verbosity: log('rewriting %s (%.2f%% live)\n' % (basename(idx_name), live_frac * 100)) for sha in idx: if live_objects.exists(sha): item_it = cat_pipe.get(hexlify(sha)) _, typ, _ = next(item_it) writer.just_write(sha, typ, b''.join(item_it)) ns.stale_files.append(idx_name) ns.stale_files.append(idx_name[:-3] + b'pack') if verbosity: progress('preserving live data (%d%% complete)\n' % ((float(collect_count) / existing_count) * 100)) # Nothing should have recreated midx/bloom yet. pack_dir = git.repo(b'objects/pack') assert (not os.path.exists(os.path.join(pack_dir, b'bup.bloom'))) assert (not glob.glob(os.path.join(pack_dir, b'*.midx'))) except BaseException as ex: with pending_raise(ex): writer.abort() # This will finally run midx. # Can only change refs (if needed) after this. writer.close() remove_stale_files(None) # In case we didn't write to the writer. if verbosity: log('discarded %d%% of objects\n' % ((existing_count - count_objects(pack_dir, verbosity)) / float(existing_count) * 100))
def receive_objects_v2(conn, junk): global suspended_w _init_session() if suspended_w: w = suspended_w suspended_w = None elif dumb_server_mode: w = git.PackWriter(objcache_maker=None) else: w = git.PackWriter() try: suggested = set() while 1: ns = conn.read(4) if not ns: w.abort() raise Exception('object read: expected length header, got EOF') n = struct.unpack('!I', ns)[0] #debug2('expecting %d bytes\n' % n) if not n: debug1('bup server: received %d object%s.\n' % (w.count, w.count!=1 and "s" or '')) fullpath = w.close(run_midx=not dumb_server_mode) w = None if fullpath: dir, name = os.path.split(fullpath) conn.write(b'%s.idx\n' % name) conn.ok() return elif n == 0xffffffff: debug2('bup server: receive-objects suspending\n') conn.ok() suspended_w = w w = None return shar = conn.read(20) crcr = struct.unpack('!I', conn.read(4))[0] n -= 20 + 4 buf = conn.read(n) # object sizes in bup are reasonably small #debug2('read %d bytes\n' % n) _check(w, n, len(buf), 'object read: expected %d bytes, got %d\n') if not dumb_server_mode: oldpack = w.exists(shar, want_source=True) if oldpack: assert(not oldpack == True) assert(oldpack.endswith(b'.idx')) (dir,name) = os.path.split(oldpack) if not (name in suggested): debug1("bup server: suggesting index %s\n" % git.shorten_hash(name).decode('ascii')) debug1("bup server: because of object %s\n" % hexstr(shar)) conn.write(b'index %s\n' % name) suggested.add(name) continue nw, crc = w._raw_write((buf,), sha=shar) _check(w, crcr, crc, 'object read: expected crc %d, got %d\n') # py2: this clause is unneeded with py3 except BaseException as ex: with pending_raise(ex): if w: w, w_tmp = None, w w_tmp.close() finally: if w: w.close() assert False # should be unreachable
def __exit__(self, type, value, traceback): with pending_raise(value, rethrow=True): self.abort_save()