def _setup(r, network):
with errctx('setup %s' % qq(network)):
with r._dbpool.xget() as conn:
with conn:
conn.execute("""
CREATE TABLE IF NOT EXISTS hosts (
hostname TEXT NON NULL PRIMARY KEY,
osladdr TEXT NON NULL
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS meta (
name TEXT NON NULL PRIMARY KEY,
value TEXT NON NULL
)
""")
ver = r._config(conn, "schemaver")
if ver == "":
ver = r.schema_ver
r._set_config(conn, "schemaver", ver)
if ver != r.schema_ver:
raise Error(
'schema version mismatch: want %s; have %s' %
(qq(r._schema_ver), qq(ver)))
dbnetwork = r._config(conn, "network")
if dbnetwork == "":
dbnetwork = network
r._set_config(conn, "network", dbnetwork)
if dbnetwork != network:
raise Error('network name mismatch: want %s; have %s' %
(qq(network), qq(dbnetwork)))
def zdump(self): # -> bytes
z = b'txn %s %s\n' % (ashex(self.tid), qq(self.status))
z += b'user %s\n' % qq(self.user)
z += b'description %s\n' % qq(self.description)
z += b'extension %s\n' % qq(self.extension_bytes)
for obj in self.objv:
z += obj.zdump()
z += b'\n'
return z
def __loaccept(n, osconn):
line = skreadline(osconn, 1024)
def reply(reply):
line = "< lonet %s %s\n" % (qq(n._network), reply)
osconn.sendall(line)
def ereply(err, tb):
e = err
if err is ErrConnRefused:
e = "connection refused" # str(ErrConnRefused) is "[Errno 111] connection refused"
reply("E %s" % qq(e))
if not xerr.well_defined(err):
err = Error("BUG", err, cause_tb=tb)
raise err
def eproto(ereason, detail):
reply("E %s" % qq(protocolError(ereason)))
raise protocolError(ereason + ": " + detail)
m = _lodial_re.match(line)
if m is None:
eproto("invalid dial request", "%s" % qq(line))
network = m.group('network').decode('string_escape')
src = m.group('src').decode('string_escape')
dst = m.group('dst').decode('string_escape')
if network != n._network:
eproto("network mismatch", "%s" % qq(network))
try:
asrc = Addr.parse(network, src)
except ValueError:
eproto("src address invalid", "%s" % qq(src))
try:
adst = Addr.parse(network, dst)
except ValueError:
eproto("dst address invalid", "%s" % qq(dst))
with errctx("%s <- %s" % (dst, src)):
try:
accept = n._vnet_accept(asrc, adst, osconn)
except Exception as e:
_, _, tb = sys.exc_info()
ereply(e, tb)
try:
reply('connected %s' % qq(accept.addr))
except Exception as e:
accept.ack.send(e)
raise
else:
accept.ack.send(None)
def new_host(n, name):
with errctx("virtnet %s: new host %s" % (qq(n._network), qq(name))):
n._vnet_newhost(name, n._registry)
# XXX check err due to subnet down
with n._hostmu:
if name in n._hostmap:
panic("announced ok but .hostMap already !empty" %
(qq(n._network), qq(name)))
host = Host(n, name)
n._hostmap[name] = host
n._nopenhosts += 1
return host
def ereply(err, tb):
e = err
if err is ErrConnRefused:
e = "connection refused" # str(ErrConnRefused) is "[Errno 111] connection refused"
reply("E %s" % qq(e))
if not xerr.well_defined(err):
err = Error("BUG", err, cause_tb=tb)
raise err
def _config(r, conn, name):
with errctx('config: get %s' % qq(name)):
rowv = query(conn, "SELECT value FROM meta WHERE name = ?", name)
if len(rowv) == 0:
return ""
if len(rowv) > 1:
raise Error("registry broken: duplicate config entries")
return rowv[0][0]
def close(h):
def autoclose():
def _():
n = h._subnet
with n._hostmu:
n._nopenHosts -= 1
if n._nopenHosts < 0:
panic("SubNetwork._nopenHosts < 0")
if n._autoclose and n._nopenHosts == 0:
n._closeWithoutHosts()
h._close_once.do(_)
defer(autoclose)
with errctx("virtnet %s: host %s: close" %
(qq(h._subnet._network), qq(h._name))):
h._shutdown()
def _(osconn):
# XXX defer wg.Done()
myaddr = addrstr4(*n._oslistener.getsockname())
peeraddr = addrstr4(*osconn.getpeername())
try:
n._loaccept(osconn)
except Exception as e:
if errcause(e) is not ErrConnRefused:
log.error("lonet %s: serve %s <- %s : %s" %
(qq(n._network), myaddr, peeraddr, e))
def join(network):
with errctx("lonet: join %s" % qq(network)):
lonet = tempfile.gettempdir() + "/lonet"
_mkdir_p(lonet, 0777 | stat.S_ISVTX)
if network != "":
netdir = lonet + "/" + network
_mkdir_p(netdir, 0700)
else:
netdir = tempfile.mkdtemp(dir=lonet)
network = os.path.basename(netdir)
registry = SQLiteRegistry(netdir + "/registry.db", network)
return _SubNetwork("lonet" + network, registry)
def __loconnect(n, osconn, src, dst):
osconn.sendall("> lonet %s dial %s %s\n" %
(qq(n._network), qq(src), qq(dst)))
line = skreadline(osconn, 1024)
m = _loreply_re.match(line)
if m is None:
raise protocolError("invalid dial reply: %s" % qq(line))
network = m.group('network').decode('string_escape')
reply = m.group('reply') # no unescape
arg = m.group('arg').decode('string_escape')
if reply == "E":
if arg == "connection refused":
raise ErrConnRefused
else:
raise Error(arg)
if reply == "connected":
pass # ok
else:
raise protocolError("invalid reply verb: %s" % qq(reply))
if network != n._network:
raise protocolError("connected, but network mismatch: %s" %
qq(network))
try:
acceptAddr = Addr.parse(network, arg)
except ValueError:
raise protocolError("connected, but accept address invalid: %s" %
qq(acceptAddr))
if acceptAddr.host != dst.host:
raise protocolError(
"connected, but accept address is for different host: %s" %
qq(acceptAddr.host))
# everything is ok
return acceptAddr
def readtxn(self):
# header
l = self._readline()
if l is None:
return None
m = _txn_re.match(l)
if m is None:
self._badline('no txn start')
tid = fromhex(m.group('tid'))
status = m.group('status')
def get(name):
l = self._readline()
if l is None or not l.startswith(b'%s ' % name):
self._badline('no %s' % name)
return strconv.unquote(l[len(name) + 1:])
user = get(b'user')
description = get(b'description')
extension = get(b'extension')
# objects
objv = []
while 1:
l = self._readline()
if l == b'':
break # empty line - end of transaction
if l is None or not l.startswith(b'obj '):
self._badline('no obj')
m = _obj_re.match(l)
if m is None:
self._badline('invalid obj entry')
obj = None # will be Object*
oid = fromhex(m.group('oid'))
from_ = m.group('from')
if m.group('delete'):
obj = ObjectDelete(oid)
elif from_:
copy_from = fromhex(from_)
obj = ObjectCopy(oid, copy_from)
else:
size = int(m.group('size'))
hashfunc = m.group('hashfunc')
hashok = fromhex(m.group('hash'))
hashonly = m.group('hashonly') is not None
data = None # see vvv
hcls = hashRegistry.get(hashfunc)
if hcls is None:
self._badline('unknown hash function %s' % qq(hashfunc))
if hashonly:
data = HashOnly(size)
else:
# XXX -> io.readfull
n = size + 1 # data LF
data = b''
while n > 0:
chunk = self._r.read(n)
data += chunk
n -= len(chunk)
self.lineno += data.count(b'\n')
self._line = None
if data[-1:] != b'\n':
raise RuntimeError('%s+%d: no LF after obj data' %
(_ioname(self._r), self.lineno))
data = data[:-1]
# verify data integrity
# TODO option to allow reading corrupted data
h = hcls()
h.update(data)
hash_ = h.digest()
if hash_ != hashok:
raise RuntimeError(
'%s+%d: data corrupt: %s = %s, expected %s' %
(_ioname(self._r), self.lineno, h.name,
ashex(hash_), ashex(hashok)))
obj = ObjectData(oid, data, hashfunc, hashok)
objv.append(obj)
return Transaction(tid, status, user, description, extension, objv)
def _badline(self, msg):
raise RuntimeError(
"%s+%d: invalid line: %s (%s)" %
(_ioname(self._r), self.lineno, msg, qq(self._line)))
def zodbdump(stor,
tidmin,
tidmax,
hashonly=False,
pretty='raw',
out=asbinstream(sys.stdout)):
def badpretty():
raise ValueError("invalid pretty format %s" % pretty)
for txn in stor.iterator(tidmin, tidmax):
# XXX .status not covered by IStorageTransactionInformation
# XXX but covered by BaseStorage.TransactionRecord
out.write(b"txn %s %s\nuser %s\ndescription %s\n" % (ashex(
txn.tid), qq(txn.status), qq(txn.user), qq(txn.description)))
# extension is saved by ZODB as either empty or as pickle dump of an object
rawext = txn_raw_extension(stor, txn)
if pretty == 'raw':
out.write(b"extension %s\n" % qq(rawext))
elif pretty == 'zpickledis':
if len(rawext) == 0:
out.write(b'extension ""\n')
else:
out.write(b"extension\n")
extf = BytesIO(rawext)
disf = BytesIO()
pickletools.dis(extf, disf)
out.write(indent(disf.getvalue(), " "))
extra = extf.read()
if len(extra) > 0:
out.write(b" + extra data %s\n" % qq(extra))
else:
badpretty()
objv = txnobjv(txn)
for obj in objv:
entry = b"obj %s " % ashex(obj.oid)
write_data = False
if obj.data is None:
entry += b"delete"
# was undo and data taken from obj.data_txn
elif obj.data_txn is not None:
entry += b"from %s" % ashex(obj.data_txn)
else:
# XXX sha1 is hardcoded for now. Dump format allows other hashes.
entry += b"%i sha1:%s" % (len(obj.data), ashex(sha1(obj.data)))
write_data = True
out.write(b(entry))
if write_data:
if hashonly:
out.write(b" -")
else:
out.write(b"\n")
if pretty == 'raw':
out.write(obj.data)
elif pretty == 'zpickledis':
# https://github.com/zopefoundation/ZODB/blob/5.6.0-55-g1226c9d35/src/ZODB/serialize.py#L24-L29
dataf = BytesIO(obj.data)
disf = BytesIO()
pickletools.dis(dataf, disf) # class
pickletools.dis(dataf, disf) # state
out.write(indent(disf.getvalue(), " "))
extra = dataf.read()
if len(extra) > 0:
out.write(b" + extra data %s\n" % qq(extra))
else:
badpretty()
out.write(b"\n")
out.write(b"\n")
def eproto(ereason, detail):
reply("E %s" % qq(protocolError(ereason)))
raise protocolError(ereason + ": " + detail)
def _set_config(r, conn, name, value):
with errctx('config: set %s = %s' % (qq(name), qq(value))):
conn.execute(
"INSERT OR REPLACE INTO meta (name, value) VALUES (?, ?)",
(name, value))
def reply(reply):
line = "< lonet %s %s\n" % (qq(n._network), reply)
osconn.sendall(line)
def __close(n, withHosts):
with errctx("virtnet %s: close" % qq(n._network)):
n.__shutdown(None, withHosts)
def _vnet_down(n, exc):
# XXX py: errctx here (go does not have) because we do not reraise .downErr in close
with errctx("virtnet %s: shutdown" % qq(n._network)):
n._shutdown(exc)
def test_quote():
testv = (
# in quoted without leading/trailing "
('', r""),
(byterange(0, 32),
br'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f'
),
('\'', r"'"),
('"', r"\""),
('ab c\ndef', r"ab c\ndef"),
('a\'c\ndef', r"a'c\ndef"),
('a\"c\ndef', r"a\"c\ndef"),
(u'a\"c\ndef', u"a\\\"c\\ndef"),
(b'a\"c\ndef', br'a\"c\ndef'),
('привет\nмир', r"привет\nмир"),
(u'привет\nмир', u"привет\\nмир"),
# invalid utf-8
(b"\xd0a", br"\xd0a"),
# non-printable utf-8
(u"\u007f\u0080\u0081\u0082\u0083\u0084\u0085\u0086\u0087",
u"\\x7f\\xc2\\x80\\xc2\\x81\\xc2\\x82\\xc2\\x83\\xc2\\x84\\xc2\\x85\\xc2\\x86\\xc2\\x87"
),
)
for tin, tquoted in testv:
# quote(in) == quoted
# in = unquote(quoted)
q = b'"' if isinstance(tquoted, bytes) else '"'
tail = b'123' if isinstance(tquoted, bytes) else '123'
tquoted = q + tquoted + q # add lead/trail "
assert quote(tin) == tquoted
assert unquote(tquoted) == tin
assert unquote_next(tquoted) == (tin, type(tin)())
assert unquote_next(tquoted + tail) == (tin, tail)
with raises(ValueError):
unquote(tquoted + tail)
# qq always gives str
assert qq(tin) == asstr(tquoted)
# also check how it works on complementary unicode/bytes input type
if isinstance(tin, bytes):
try:
tin = tin.decode('utf-8')
except UnicodeDecodeError:
# some inputs are not valid UTF-8
continue
tquoted = tquoted.decode('utf-8')
tail = tail.decode('utf-8')
else:
# tin was unicode
tin = tin.encode('utf-8')
tquoted = tquoted.encode('utf-8')
tail = tail.encode('utf-8')
assert quote(tin) == tquoted
assert unquote(tquoted) == tin
assert unquote_next(tquoted) == (tin, type(tin)())
assert unquote_next(tquoted + tail) == (tin, tail)
with raises(ValueError):
unquote(tquoted + tail)
# qq always gives str
assert qq(tin) == asstr(tquoted)
def main():
sb = b("привет b")
su = u("привет u")
print("print(qq(b)):", qq(sb))
print("print(qq(u)):", qq(su))