def _setup(r, network): with errctx('setup %s' % qq(network)): with r._dbpool.xget() as conn: with conn: conn.execute(""" CREATE TABLE IF NOT EXISTS hosts ( hostname TEXT NON NULL PRIMARY KEY, osladdr TEXT NON NULL ) """) conn.execute(""" CREATE TABLE IF NOT EXISTS meta ( name TEXT NON NULL PRIMARY KEY, value TEXT NON NULL ) """) ver = r._config(conn, "schemaver") if ver == "": ver = r.schema_ver r._set_config(conn, "schemaver", ver) if ver != r.schema_ver: raise Error( 'schema version mismatch: want %s; have %s' % (qq(r._schema_ver), qq(ver))) dbnetwork = r._config(conn, "network") if dbnetwork == "": dbnetwork = network r._set_config(conn, "network", dbnetwork) if dbnetwork != network: raise Error('network name mismatch: want %s; have %s' % (qq(network), qq(dbnetwork)))
def zdump(self): # -> bytes z = b'txn %s %s\n' % (ashex(self.tid), qq(self.status)) z += b'user %s\n' % qq(self.user) z += b'description %s\n' % qq(self.description) z += b'extension %s\n' % qq(self.extension_bytes) for obj in self.objv: z += obj.zdump() z += b'\n' return z
def __loaccept(n, osconn): line = skreadline(osconn, 1024) def reply(reply): line = "< lonet %s %s\n" % (qq(n._network), reply) osconn.sendall(line) def ereply(err, tb): e = err if err is ErrConnRefused: e = "connection refused" # str(ErrConnRefused) is "[Errno 111] connection refused" reply("E %s" % qq(e)) if not xerr.well_defined(err): err = Error("BUG", err, cause_tb=tb) raise err def eproto(ereason, detail): reply("E %s" % qq(protocolError(ereason))) raise protocolError(ereason + ": " + detail) m = _lodial_re.match(line) if m is None: eproto("invalid dial request", "%s" % qq(line)) network = m.group('network').decode('string_escape') src = m.group('src').decode('string_escape') dst = m.group('dst').decode('string_escape') if network != n._network: eproto("network mismatch", "%s" % qq(network)) try: asrc = Addr.parse(network, src) except ValueError: eproto("src address invalid", "%s" % qq(src)) try: adst = Addr.parse(network, dst) except ValueError: eproto("dst address invalid", "%s" % qq(dst)) with errctx("%s <- %s" % (dst, src)): try: accept = n._vnet_accept(asrc, adst, osconn) except Exception as e: _, _, tb = sys.exc_info() ereply(e, tb) try: reply('connected %s' % qq(accept.addr)) except Exception as e: accept.ack.send(e) raise else: accept.ack.send(None)
def new_host(n, name): with errctx("virtnet %s: new host %s" % (qq(n._network), qq(name))): n._vnet_newhost(name, n._registry) # XXX check err due to subnet down with n._hostmu: if name in n._hostmap: panic("announced ok but .hostMap already !empty" % (qq(n._network), qq(name))) host = Host(n, name) n._hostmap[name] = host n._nopenhosts += 1 return host
def ereply(err, tb): e = err if err is ErrConnRefused: e = "connection refused" # str(ErrConnRefused) is "[Errno 111] connection refused" reply("E %s" % qq(e)) if not xerr.well_defined(err): err = Error("BUG", err, cause_tb=tb) raise err
def _config(r, conn, name): with errctx('config: get %s' % qq(name)): rowv = query(conn, "SELECT value FROM meta WHERE name = ?", name) if len(rowv) == 0: return "" if len(rowv) > 1: raise Error("registry broken: duplicate config entries") return rowv[0][0]
def close(h): def autoclose(): def _(): n = h._subnet with n._hostmu: n._nopenHosts -= 1 if n._nopenHosts < 0: panic("SubNetwork._nopenHosts < 0") if n._autoclose and n._nopenHosts == 0: n._closeWithoutHosts() h._close_once.do(_) defer(autoclose) with errctx("virtnet %s: host %s: close" % (qq(h._subnet._network), qq(h._name))): h._shutdown()
def _(osconn): # XXX defer wg.Done() myaddr = addrstr4(*n._oslistener.getsockname()) peeraddr = addrstr4(*osconn.getpeername()) try: n._loaccept(osconn) except Exception as e: if errcause(e) is not ErrConnRefused: log.error("lonet %s: serve %s <- %s : %s" % (qq(n._network), myaddr, peeraddr, e))
def join(network): with errctx("lonet: join %s" % qq(network)): lonet = tempfile.gettempdir() + "/lonet" _mkdir_p(lonet, 0777 | stat.S_ISVTX) if network != "": netdir = lonet + "/" + network _mkdir_p(netdir, 0700) else: netdir = tempfile.mkdtemp(dir=lonet) network = os.path.basename(netdir) registry = SQLiteRegistry(netdir + "/registry.db", network) return _SubNetwork("lonet" + network, registry)
def __loconnect(n, osconn, src, dst): osconn.sendall("> lonet %s dial %s %s\n" % (qq(n._network), qq(src), qq(dst))) line = skreadline(osconn, 1024) m = _loreply_re.match(line) if m is None: raise protocolError("invalid dial reply: %s" % qq(line)) network = m.group('network').decode('string_escape') reply = m.group('reply') # no unescape arg = m.group('arg').decode('string_escape') if reply == "E": if arg == "connection refused": raise ErrConnRefused else: raise Error(arg) if reply == "connected": pass # ok else: raise protocolError("invalid reply verb: %s" % qq(reply)) if network != n._network: raise protocolError("connected, but network mismatch: %s" % qq(network)) try: acceptAddr = Addr.parse(network, arg) except ValueError: raise protocolError("connected, but accept address invalid: %s" % qq(acceptAddr)) if acceptAddr.host != dst.host: raise protocolError( "connected, but accept address is for different host: %s" % qq(acceptAddr.host)) # everything is ok return acceptAddr
def readtxn(self): # header l = self._readline() if l is None: return None m = _txn_re.match(l) if m is None: self._badline('no txn start') tid = fromhex(m.group('tid')) status = m.group('status') def get(name): l = self._readline() if l is None or not l.startswith(b'%s ' % name): self._badline('no %s' % name) return strconv.unquote(l[len(name) + 1:]) user = get(b'user') description = get(b'description') extension = get(b'extension') # objects objv = [] while 1: l = self._readline() if l == b'': break # empty line - end of transaction if l is None or not l.startswith(b'obj '): self._badline('no obj') m = _obj_re.match(l) if m is None: self._badline('invalid obj entry') obj = None # will be Object* oid = fromhex(m.group('oid')) from_ = m.group('from') if m.group('delete'): obj = ObjectDelete(oid) elif from_: copy_from = fromhex(from_) obj = ObjectCopy(oid, copy_from) else: size = int(m.group('size')) hashfunc = m.group('hashfunc') hashok = fromhex(m.group('hash')) hashonly = m.group('hashonly') is not None data = None # see vvv hcls = hashRegistry.get(hashfunc) if hcls is None: self._badline('unknown hash function %s' % qq(hashfunc)) if hashonly: data = HashOnly(size) else: # XXX -> io.readfull n = size + 1 # data LF data = b'' while n > 0: chunk = self._r.read(n) data += chunk n -= len(chunk) self.lineno += data.count(b'\n') self._line = None if data[-1:] != b'\n': raise RuntimeError('%s+%d: no LF after obj data' % (_ioname(self._r), self.lineno)) data = data[:-1] # verify data integrity # TODO option to allow reading corrupted data h = hcls() h.update(data) hash_ = h.digest() if hash_ != hashok: raise RuntimeError( '%s+%d: data corrupt: %s = %s, expected %s' % (_ioname(self._r), self.lineno, h.name, ashex(hash_), ashex(hashok))) obj = ObjectData(oid, data, hashfunc, hashok) objv.append(obj) return Transaction(tid, status, user, description, extension, objv)
def _badline(self, msg): raise RuntimeError( "%s+%d: invalid line: %s (%s)" % (_ioname(self._r), self.lineno, msg, qq(self._line)))
def zodbdump(stor, tidmin, tidmax, hashonly=False, pretty='raw', out=asbinstream(sys.stdout)): def badpretty(): raise ValueError("invalid pretty format %s" % pretty) for txn in stor.iterator(tidmin, tidmax): # XXX .status not covered by IStorageTransactionInformation # XXX but covered by BaseStorage.TransactionRecord out.write(b"txn %s %s\nuser %s\ndescription %s\n" % (ashex( txn.tid), qq(txn.status), qq(txn.user), qq(txn.description))) # extension is saved by ZODB as either empty or as pickle dump of an object rawext = txn_raw_extension(stor, txn) if pretty == 'raw': out.write(b"extension %s\n" % qq(rawext)) elif pretty == 'zpickledis': if len(rawext) == 0: out.write(b'extension ""\n') else: out.write(b"extension\n") extf = BytesIO(rawext) disf = BytesIO() pickletools.dis(extf, disf) out.write(indent(disf.getvalue(), " ")) extra = extf.read() if len(extra) > 0: out.write(b" + extra data %s\n" % qq(extra)) else: badpretty() objv = txnobjv(txn) for obj in objv: entry = b"obj %s " % ashex(obj.oid) write_data = False if obj.data is None: entry += b"delete" # was undo and data taken from obj.data_txn elif obj.data_txn is not None: entry += b"from %s" % ashex(obj.data_txn) else: # XXX sha1 is hardcoded for now. Dump format allows other hashes. entry += b"%i sha1:%s" % (len(obj.data), ashex(sha1(obj.data))) write_data = True out.write(b(entry)) if write_data: if hashonly: out.write(b" -") else: out.write(b"\n") if pretty == 'raw': out.write(obj.data) elif pretty == 'zpickledis': # https://github.com/zopefoundation/ZODB/blob/5.6.0-55-g1226c9d35/src/ZODB/serialize.py#L24-L29 dataf = BytesIO(obj.data) disf = BytesIO() pickletools.dis(dataf, disf) # class pickletools.dis(dataf, disf) # state out.write(indent(disf.getvalue(), " ")) extra = dataf.read() if len(extra) > 0: out.write(b" + extra data %s\n" % qq(extra)) else: badpretty() out.write(b"\n") out.write(b"\n")
def eproto(ereason, detail): reply("E %s" % qq(protocolError(ereason))) raise protocolError(ereason + ": " + detail)
def _set_config(r, conn, name, value): with errctx('config: set %s = %s' % (qq(name), qq(value))): conn.execute( "INSERT OR REPLACE INTO meta (name, value) VALUES (?, ?)", (name, value))
def reply(reply): line = "< lonet %s %s\n" % (qq(n._network), reply) osconn.sendall(line)
def __close(n, withHosts): with errctx("virtnet %s: close" % qq(n._network)): n.__shutdown(None, withHosts)
def _vnet_down(n, exc): # XXX py: errctx here (go does not have) because we do not reraise .downErr in close with errctx("virtnet %s: shutdown" % qq(n._network)): n._shutdown(exc)
def test_quote(): testv = ( # in quoted without leading/trailing " ('', r""), (byterange(0, 32), br'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f' ), ('\'', r"'"), ('"', r"\""), ('ab c\ndef', r"ab c\ndef"), ('a\'c\ndef', r"a'c\ndef"), ('a\"c\ndef', r"a\"c\ndef"), (u'a\"c\ndef', u"a\\\"c\\ndef"), (b'a\"c\ndef', br'a\"c\ndef'), ('привет\nмир', r"привет\nмир"), (u'привет\nмир', u"привет\\nмир"), # invalid utf-8 (b"\xd0a", br"\xd0a"), # non-printable utf-8 (u"\u007f\u0080\u0081\u0082\u0083\u0084\u0085\u0086\u0087", u"\\x7f\\xc2\\x80\\xc2\\x81\\xc2\\x82\\xc2\\x83\\xc2\\x84\\xc2\\x85\\xc2\\x86\\xc2\\x87" ), ) for tin, tquoted in testv: # quote(in) == quoted # in = unquote(quoted) q = b'"' if isinstance(tquoted, bytes) else '"' tail = b'123' if isinstance(tquoted, bytes) else '123' tquoted = q + tquoted + q # add lead/trail " assert quote(tin) == tquoted assert unquote(tquoted) == tin assert unquote_next(tquoted) == (tin, type(tin)()) assert unquote_next(tquoted + tail) == (tin, tail) with raises(ValueError): unquote(tquoted + tail) # qq always gives str assert qq(tin) == asstr(tquoted) # also check how it works on complementary unicode/bytes input type if isinstance(tin, bytes): try: tin = tin.decode('utf-8') except UnicodeDecodeError: # some inputs are not valid UTF-8 continue tquoted = tquoted.decode('utf-8') tail = tail.decode('utf-8') else: # tin was unicode tin = tin.encode('utf-8') tquoted = tquoted.encode('utf-8') tail = tail.encode('utf-8') assert quote(tin) == tquoted assert unquote(tquoted) == tin assert unquote_next(tquoted) == (tin, type(tin)()) assert unquote_next(tquoted + tail) == (tin, tail) with raises(ValueError): unquote(tquoted + tail) # qq always gives str assert qq(tin) == asstr(tquoted)
def main(): sb = b("привет b") su = u("привет u") print("print(qq(b)):", qq(sb)) print("print(qq(u)):", qq(su))