def check_midx(name): nicename = git.repo_rel(name) log('Checking %s.\n' % path_msg(nicename)) try: ix = git.open_idx(name) except git.GitError as e: add_error('%s: %s' % (path_msg(name), e)) return for count,subname in enumerate(ix.idxnames): sub = git.open_idx(os.path.join(os.path.dirname(name), subname)) for ecount,e in enumerate(sub): if not (ecount % 1234): qprogress(' %d/%d: %s %d/%d\r' % (count, len(ix.idxnames), git.shorten_hash(subname).decode('ascii'), ecount, len(sub))) if not sub.exists(e): add_error("%s: %s: %s missing from idx" % (path_msg(nicename), git.shorten_hash(subname).decode('ascii'), hexstr(e))) if not ix.exists(e): add_error("%s: %s: %s missing from midx" % (path_msg(nicename), git.shorten_hash(subname).decode('ascii'), hexstr(e))) prev = None for ecount,e in enumerate(ix): if not (ecount % 1234): qprogress(' Ordering: %d/%d\r' % (ecount, len(ix))) if e and prev and not e >= prev: add_error('%s: ordering error: %s < %s' % (nicename, hexstr(e), hexstr(prev))) prev = e
def _suggest_packs(self): ob = self._busy if ob: assert(ob == 'receive-objects-v2') self.conn.write('\xff\xff\xff\xff') # suspend receive-objects-v2 suggested = [] for line in linereader(self.conn): if not line: break debug2('%s\n' % line) if line.startswith('index '): idx = line[6:] debug1('client: received index suggestion: %s\n' % git.shorten_hash(idx)) suggested.append(idx) else: assert(line.endswith('.idx')) debug1('client: completed writing pack, idx: %s\n' % git.shorten_hash(line)) suggested.append(line) self.check_ok() if ob: self._busy = None idx = None for idx in suggested: self.sync_index(idx) git.auto_midx(self.cachedir) if ob: self._busy = ob self.conn.write('%s\n' % ob) return idx
def _suggest_packs(self): ob = self._busy if ob: assert(ob == b'receive-objects-v2') self.conn.write(b'\xff\xff\xff\xff') # suspend receive-objects-v2 suggested = [] for line in linereader(self.conn): if not line: break debug2('%r\n' % line) if line.startswith(b'index '): idx = line[6:] debug1('client: received index suggestion: %s\n' % git.shorten_hash(idx).decode('ascii')) suggested.append(idx) else: assert(line.endswith(b'.idx')) debug1('client: completed writing pack, idx: %s\n' % git.shorten_hash(line).decode('ascii')) suggested.append(line) self.check_ok() if ob: self._busy = None idx = None for idx in suggested: self.sync_index(idx) git.auto_midx(self.cachedir) if ob: self._busy = ob self.conn.write(b'%s\n' % ob) return idx
def check_midx(name): nicename = git.repo_rel(name) log('Checking %s.\n' % nicename) try: ix = git.open_idx(name) except git.GitError as e: add_error('%s: %s' % (name, e)) return for count,subname in enumerate(ix.idxnames): sub = git.open_idx(os.path.join(os.path.dirname(name), subname)) for ecount,e in enumerate(sub): if not (ecount % 1234): qprogress(' %d/%d: %s %d/%d\r' % (count, len(ix.idxnames), git.shorten_hash(subname), ecount, len(sub))) if not sub.exists(e): add_error("%s: %s: %s missing from idx" % (nicename, git.shorten_hash(subname), str(e).encode('hex'))) if not ix.exists(e): add_error("%s: %s: %s missing from midx" % (nicename, git.shorten_hash(subname), str(e).encode('hex'))) prev = None for ecount,e in enumerate(ix): if not (ecount % 1234): qprogress(' Ordering: %d/%d\r' % (ecount, len(ix))) if not e >= prev: add_error('%s: ordering error: %s < %s' % (nicename, str(e).encode('hex'), str(prev).encode('hex'))) prev = e
def receive_objects_v2(conn, junk): global suspended_w _init_session() suggested = set() if suspended_w: w = suspended_w suspended_w = None else: if dumb_server_mode: w = git.PackWriter(objcache_maker=None) else: w = git.PackWriter() while 1: ns = conn.read(4) if not ns: w.abort() raise Exception('object read: expected length header, got EOF\n') n = struct.unpack('!I', ns)[0] #debug2('expecting %d bytes\n' % n) if not n: debug1('bup server: received %d object%s.\n' % (w.count, w.count!=1 and "s" or '')) fullpath = w.close(run_midx=not dumb_server_mode) if fullpath: (dir, name) = os.path.split(fullpath) conn.write('%s.idx\n' % name) conn.ok() return elif n == 0xffffffff: debug2('bup server: receive-objects suspended.\n') suspended_w = w conn.ok() return shar = conn.read(20) crcr = struct.unpack('!I', conn.read(4))[0] n -= 20 + 4 buf = conn.read(n) # object sizes in bup are reasonably small #debug2('read %d bytes\n' % n) _check(w, n, len(buf), 'object read: expected %d bytes, got %d\n') if not dumb_server_mode: oldpack = w.exists(shar, want_source=True) if oldpack: assert(not oldpack == True) assert(oldpack.endswith('.idx')) (dir,name) = os.path.split(oldpack) if not (name in suggested): debug1("bup server: suggesting index %s\n" % git.shorten_hash(name)) debug1("bup server: because of object %s\n" % shar.encode('hex')) conn.write('index %s\n' % name) suggested.add(name) continue nw, crc = w._raw_write((buf,), sha=shar) _check(w, crcr, crc, 'object read: expected crc %d, got %d\n')
def receive_objects_v2(conn, junk): global suspended_w git.check_repo_or_die() suggested = set() if suspended_w: w = suspended_w suspended_w = None else: if dumb_server_mode: w = git.PackWriter(objcache_maker=None) else: w = git.PackWriter() while 1: ns = conn.read(4) if not ns: w.abort() raise Exception('object read: expected length header, got EOF\n') n = struct.unpack('!I', ns)[0] #debug2('expecting %d bytes\n' % n) if not n: debug1('bup server: received %d object%s.\n' % (w.count, w.count!=1 and "s" or '')) fullpath = w.close(run_midx=not dumb_server_mode) if fullpath: (dir, name) = os.path.split(fullpath) conn.write('%s.idx\n' % name) conn.ok() return elif n == 0xffffffff: debug2('bup server: receive-objects suspended.\n') suspended_w = w conn.ok() return shar = conn.read(20) crcr = struct.unpack('!I', conn.read(4))[0] n -= 20 + 4 buf = conn.read(n) # object sizes in bup are reasonably small #debug2('read %d bytes\n' % n) _check(w, n, len(buf), 'object read: expected %d bytes, got %d\n') if not dumb_server_mode: oldpack = w.exists(shar, want_source=True) if oldpack: assert(not oldpack == True) assert(oldpack.endswith('.idx')) (dir,name) = os.path.split(oldpack) if not (name in suggested): debug1("bup server: suggesting index %s\n" % git.shorten_hash(name)) debug1("bup server: because of object %s\n" % shar.encode('hex')) conn.write('index %s\n' % name) suggested.add(name) continue nw, crc = w._raw_write((buf,), sha=shar) _check(w, crcr, crc, 'object read: expected crc %d, got %d\n')
def check_midx(name): nicename = git.repo_rel(name) log('Checking %s.\n' % nicename) try: ix = git.open_idx(name) except git.GitError, e: add_error('%s: %s' % (name, e)) return for count, subname in enumerate(ix.idxnames): sub = git.open_idx(os.path.join(os.path.dirname(name), subname)) for ecount, e in enumerate(sub): if not (ecount % 1234): qprogress(' %d/%d: %s %d/%d\r' % (count, len( ix.idxnames), git.shorten_hash(subname), ecount, len(sub))) if not sub.exists(e): add_error("%s: %s: %s missing from idx" % (nicename, git.shorten_hash(subname), str(e).encode('hex'))) if not ix.exists(e): add_error("%s: %s: %s missing from midx" % (nicename, git.shorten_hash(subname), str(e).encode('hex'))) prev = None for ecount, e in enumerate(ix): if not (ecount % 1234): qprogress(' Ordering: %d/%d\r' % (ecount, len(ix))) if not e >= prev: add_error( '%s: ordering error: %s < %s' %
def check_midx(name): nicename = git.repo_rel(name) log('Checking %s.\n' % nicename) try: ix = git.open_idx(name) except git.GitError, e: add_error('%s: %s' % (name, e)) return for count,subname in enumerate(ix.idxnames): sub = git.open_idx(os.path.join(os.path.dirname(name), subname)) for ecount,e in enumerate(sub): if not (ecount % 1234): qprogress(' %d/%d: %s %d/%d\r' % (count, len(ix.idxnames), git.shorten_hash(subname), ecount, len(sub))) if not sub.exists(e): add_error("%s: %s: %s missing from idx" % (nicename, git.shorten_hash(subname), str(e).encode('hex'))) if not ix.exists(e): add_error("%s: %s: %s missing from midx" % (nicename, git.shorten_hash(subname), str(e).encode('hex'))) prev = None for ecount,e in enumerate(ix): if not (ecount % 1234): qprogress(' Ordering: %d/%d\r' % (ecount, len(ix))) if not e >= prev: add_error('%s: ordering error: %s < %s' % (nicename,
def check_midx(name): nicename = git.repo_rel(name) log("Checking %s.\n" % nicename) try: ix = git.open_idx(name) except git.GitError, e: add_error("%s: %s" % (name, e)) return for count, subname in enumerate(ix.idxnames): sub = git.open_idx(os.path.join(os.path.dirname(name), subname)) for ecount, e in enumerate(sub): if not (ecount % 1234): qprogress( " %d/%d: %s %d/%d\r" % (count, len(ix.idxnames), git.shorten_hash(subname), ecount, len(sub)) ) if not sub.exists(e): add_error("%s: %s: %s missing from idx" % (nicename, git.shorten_hash(subname), str(e).encode("hex"))) if not ix.exists(e): add_error("%s: %s: %s missing from midx" % (nicename, git.shorten_hash(subname), str(e).encode("hex"))) prev = None for ecount, e in enumerate(ix): if not (ecount % 1234): qprogress(" Ordering: %d/%d\r" % (ecount, len(ix))) if not e >= prev: add_error("%s: ordering error: %s < %s" % (nicename, str(e).encode("hex"), str(prev).encode("hex"))) prev = e _first = None
def receive_objects_v2(self, junk): self.init_session() suggested = set() if self.suspended: self.suspended = False else: if self.dumb_server_mode: objcache_maker = lambda: None else: objcache_maker = None # FIXME: this goes together with the direct accesses below self.repo._ensure_packwriter() while 1: ns = self.conn.read(4) if not ns: self.repo.abort_writing() raise Exception( 'object read: expected length header, got EOF\n') n = struct.unpack('!I', ns)[0] #debug2('expecting %d bytes\n' % n) if not n: # FIXME: don't be lazy and count ourselves, or something, at least # don't access self.repo internals debug1('bup server: received %d object%s.\n' % (self.repo._packwriter.count, self.repo._packwriter.count != 1 and "s" or '')) fullpath = self.repo.finish_writing( run_midx=not self.dumb_server_mode) if fullpath: (dir, name) = os.path.split(fullpath) self.conn.write(b'%s.idx\n' % name) self.conn.ok() return elif n == 0xffffffff: debug2('bup server: receive-objects suspended.\n') self.suspended = True self.conn.ok() return shar = self.conn.read(20) crcr = struct.unpack('!I', self.conn.read(4))[0] n -= 20 + 4 buf = self.conn.read(n) # object sizes in bup are reasonably small #debug2('read %d bytes\n' % n) self._check(n, len(buf), 'object read: expected %d bytes, got %d\n') if not self.dumb_server_mode: result = self.repo.exists(shar, want_source=True) if result: oldpack = result.pack assert (oldpack.endswith(b'.idx')) (dir, name) = os.path.split(oldpack) if not (name in suggested): debug1("bup server: suggesting index %s\n" % git.shorten_hash(name).decode('ascii')) debug1("bup server: because of object %s\n" % hexstr(shar)) self.conn.write(b'index %s\n' % name) suggested.add(name) continue # FIXME: figure out the right abstraction for this; or better yet, # make the protocol aware of the object type nw, crc = self.repo._packwriter._raw_write((buf, ), sha=shar) self._check(crcr, crc, 'object read: expected crc %d, got %d\n')