async def process_routine(event):
try:
conn = event.connection
msg = event.message
_logger.debug('Request received: %r' % (dump(msg),))
try:
reply = await func(self, msg)
await conn.protocol.reply_to(conn,
reply,
msg,
self)
_logger.debug('Send reply: %r' % (dump(reply),))
except (IOError, ConnectionResetException):
pass
except Exception as exc:
_logger.warning('Request process failed:', exc_info=True)
await conn.protocol.reply_to(conn,
d.message_error(
err_type=d.RUNTIME_ERROR,
details=str(exc).encode('utf-8')
),
msg,
self)
except (IOError, ConnectionResetException):
pass
except Exception:
_logger.warning('Unexpected error:', exc_info=True)
def testOxm(self):
for ofp in (openflow13, openflow14):
fm = ofp.ofp_flow_mod.new(priority = ofp.OFP_DEFAULT_PRIORITY, command = ofp.OFPFC_ADD, buffer_id = ofp.OFP_NO_BUFFER)
fm.cookie = 0x67843512
fm.match = ofp.ofp_match_oxm.new()
fm.match.oxm_fields.append(ofp.create_oxm(ofp.OXM_OF_ETH_DST, b'\x06\x00\x0c\x15\x45\x99'))
fm.match.oxm_fields.append(ofp.create_oxm(ofp.OXM_OF_ETH_TYPE, common.ETHERTYPE_IP))
fm.match.oxm_fields.append(ofp.create_oxm(ofp.OXM_OF_IP_PROTO, 6))
fm.match.oxm_fields.append(ofp.create_oxm(ofp.OXM_OF_IPV4_SRC_W, [192,168,1,0], [255,255,255,0]))
apply = ofp.ofp_instruction_actions.new(type = ofp.OFPIT_APPLY_ACTIONS)
apply.actions.append(ofp.ofp_action_set_field.new(field = ofp.create_oxm(ofp.OXM_OF_IPV4_SRC, [202, 102, 0, 37])))
apply.actions.append(ofp.ofp_action_set_queue.new(queue_id = 1))
fm.instructions.append(apply)
write = ofp.ofp_instruction_actions.new(type = ofp.OFPIT_WRITE_ACTIONS)
write.actions.append(ofp.ofp_action_output.new(port = 7))
fm.instructions.append(write)
goto = ofp.ofp_instruction_goto_table.new(table_id = 1)
fm.instructions.append(goto)
s = fm._tobytes()
r = common.ofp_msg.parse(s)
self.assertTrue(r is not None, 'Cannot parse message')
obj2, size = r
self.assertEqual(size, len(s), 'Cannot parse message')
print(json.dumps(dump(fm, tostr=True), indent=2))
print(json.dumps(dump(obj2, tostr=True), indent=2))
self.assertEqual(dump(fm), dump(obj2), 'message changed after parsing')
def testOxm(self):
fm = openflow13.ofp_flow_mod.new(priority = openflow13.OFP_DEFAULT_PRIORITY, command = openflow13.OFPFC_ADD, buffer_id = openflow13.OFP_NO_BUFFER)
fm.cookie = 0x67843512
fm.match = openflow13.ofp_match_oxm.new()
fm.match.oxm_fields.append(openflow13.create_oxm(openflow13.OXM_OF_ETH_DST, b'\x06\x00\x0c\x15\x45\x99'))
fm.match.oxm_fields.append(openflow13.create_oxm(openflow13.OXM_OF_ETH_TYPE, common.ETHERTYPE_IP))
fm.match.oxm_fields.append(openflow13.create_oxm(openflow13.OXM_OF_IP_PROTO, 6))
fm.match.oxm_fields.append(openflow13.create_oxm(openflow13.OXM_OF_IPV4_SRC_W, [192,168,1,0], [255,255,255,0]))
apply = openflow13.ofp_instruction_actions.new(type = openflow13.OFPIT_APPLY_ACTIONS)
apply.actions.append(openflow13.ofp_action_set_field.new(field = openflow13.create_oxm(openflow13.OXM_OF_IPV4_SRC, [202, 102, 0, 37])))
apply.actions.append(openflow13.ofp_action_set_queue.new(queue_id = 1))
fm.instructions.append(apply)
write = openflow13.ofp_instruction_actions.new(type = openflow13.OFPIT_WRITE_ACTIONS)
write.actions.append(openflow13.ofp_action_output.new(port = 7))
fm.instructions.append(write)
goto = openflow13.ofp_instruction_goto_table.new(table_id = 1)
fm.instructions.append(goto)
s = fm._tobytes()
r = common.ofp_msg.parse(s)
self.assertTrue(r is not None, 'Cannot parse message')
obj2, size = r
self.assertEqual(size, len(s), 'Cannot parse message')
pprint(dump(fm))
pprint(dump(obj2))
self.assertEqual(dump(fm), dump(obj2), 'message changed after parsing')
def main(self):
self.apiroutine.subroutine(self.watcher(), False, daemon=True)
up = ZooKeeperConnectionStateEvent.createMatcher(
ZooKeeperConnectionStateEvent.UP, self.client)
notconn = ZooKeeperConnectionStateEvent.createMatcher(
ZooKeeperConnectionStateEvent.NOTCONNECTED, self.client)
yield (up, notconn)
if self.apiroutine.matcher is notconn:
print('Not connected')
return
else:
print('Connection is up: %r' % (self.client, ))
# Handshake
for m in self.protocol.handshake(
self.client,
zk.ConnectRequest(
timeOut=int(self.sessiontimeout * 1000),
passwd=b'\x00' * 16, # Why is it necessary...
),
self.apiroutine,
[]):
yield m
for m in self.protocol.requests(
self.client,
[zk.create(b'/vlcptest', b'test'),
zk.getdata(b'/vlcptest', True)], self.apiroutine):
yield m
pprint(dump(self.apiroutine.retvalue[0]))
for m in self.apiroutine.waitWithTimeout(0.2):
yield m
for m in self.protocol.requests(
self.client,
[zk.delete(b'/vlcptest'),
zk.getdata(b'/vlcptest', watch=True)], self.apiroutine):
yield m
pprint(dump(self.apiroutine.retvalue[0]))
for m in self.protocol.requests(self.client, [
zk.multi(zk.multi_create(b'/vlcptest2', b'test'),
zk.multi_create(b'/vlcptest2/subtest', 'test2')),
zk.getchildren2(b'/vlcptest2', True)
], self.apiroutine):
yield m
pprint(dump(self.apiroutine.retvalue[0]))
for m in self.protocol.requests(self.client, [
zk.multi(zk.multi_delete(b'/vlcptest2/subtest'),
zk.multi_delete(b'/vlcptest2')),
zk.getchildren2(b'/vlcptest2', True)
], self.apiroutine):
yield m
pprint(dump(self.apiroutine.retvalue[0]))
def testDefs13(self):
for k in dir(openflow13):
attr = getattr(openflow13, k)
if isinstance(attr, nstruct) and not attr in self.exclude and not k.startswith('ofp_oxm_') and not k.startswith('nxm_') and not hasattr(ethernet, k):
if not attr.subclasses:
self.assertEqual(k, repr(attr), k + ' has different name: ' + repr(attr))
print(k, repr(attr))
obj = attr.new()
s = obj._tobytes()
r = attr.parse(s)
self.assertTrue(r is not None, repr(attr) + ' failed to parse')
obj2, size = r
self.assertEqual(size, len(s), repr(attr) + ' failed to parse')
self.assertEqual(dump(obj), dump(obj2), repr(attr) + ' changed after parsing')
def testDefs10(self):
for k in dir(openflow10):
attr = getattr(openflow10, k)
if isinstance(attr, nstruct) and not attr in self.exclude and not k.startswith('nxm_') and not hasattr(ethernet, k):
if not attr.subclasses:
self.assertEqual(k, repr(attr), k + ' has different name: ' + repr(attr))
print(k, repr(attr))
obj = attr.new()
s = obj._tobytes()
r = attr.parse(s)
self.assertTrue(r is not None, repr(attr) + ' failed to parse')
obj2, size = r
self.assertEqual(size, len(s), repr(attr) + ' failed to parse')
self.assertEqual(dump(obj), dump(obj2), repr(attr) + ' changed after parsing')
def main(self):
def _watch(w):
for m in w.wait(self.apiroutine):
yield m
print('Watcher returns:', dump(self.apiroutine.retvalue))
def _watchall(watchers):
for w in watchers:
if w is not None:
self.apiroutine.subroutine(_watch(w))
self.apiroutine.subroutine(self.watcher(), False, daemon=True)
up = ZooKeeperSessionStateChanged.createMatcher(
ZooKeeperSessionStateChanged.CREATED, self.client)
yield (up, )
print('Connection is up: %r' % (self.client.currentserver, ))
for m in self.client.requests(
[zk.create(b'/vlcptest', b'test'),
zk.getdata(b'/vlcptest', True)], self.apiroutine):
yield m
print(self.apiroutine.retvalue)
pprint(dump(self.apiroutine.retvalue[0]))
_watchall(self.apiroutine.retvalue[3])
for m in self.apiroutine.waitWithTimeout(0.2):
yield m
for m in self.client.requests(
[zk.delete(b'/vlcptest'),
zk.getdata(b'/vlcptest', watch=True)], self.apiroutine):
yield m
print(self.apiroutine.retvalue)
pprint(dump(self.apiroutine.retvalue[0]))
_watchall(self.apiroutine.retvalue[3])
for m in self.client.requests([
zk.multi(zk.multi_create(b'/vlcptest2', b'test'),
zk.multi_create(b'/vlcptest2/subtest', 'test2')),
zk.getchildren2(b'/vlcptest2', True)
], self.apiroutine):
yield m
print(self.apiroutine.retvalue)
pprint(dump(self.apiroutine.retvalue[0]))
_watchall(self.apiroutine.retvalue[3])
for m in self.client.requests([
zk.multi(zk.multi_delete(b'/vlcptest2/subtest'),
zk.multi_delete(b'/vlcptest2')),
zk.getchildren2(b'/vlcptest2', True)
], self.apiroutine):
yield m
print(self.apiroutine.retvalue)
pprint(dump(self.apiroutine.retvalue[0]))
_watchall(self.apiroutine.retvalue[3])
def querymultipart(self,
request,
connection,
container,
raiseonerror=True):
for m in connection.write(self.formatrequest(request, connection),
False):
yield m
reply = self.replymatcher(request, connection)
conndown = self.statematcher(connection)
messages = []
while True:
yield (reply, conndown)
if container.matcher is conndown:
raise OpenflowProtocolException(
'Connection is down before reply received')
msg = container.event.message
messages.append(msg)
if msg.header.type == common.OFPT_ERROR or not (
msg.flags & common.OFPSF_REPLY_MORE):
if msg.header.type == common.OFPT_ERROR and raiseonerror:
container.openflow_reply = messages
raise OpenflowErrorResultException(
'An error message is returned: ' + repr(dump(msg)))
break
container.openflow_reply = messages
def batch(self, requests, connection, container, raiseonerror = True):
for r in requests:
self.assignxid(r, connection)
replymatchers = dict((self.replymatcher(r, connection), r) for r in requests)
replydict = {}
replymessages = []
conndown = self.statematcher(connection)
firsterror = [None]
def callback(event, matcher):
if matcher is conndown:
raise OpenflowProtocolException('Connection is down before reply received')
msg = event.message
if event.iserror and not firsterror[0]:
firsterror[0] = msg
replydict.setdefault(matcher, []).append(msg)
replymessages.append(msg)
def batchprocess():
for r in requests:
for m in connection.write(self.formatrequest(r, connection, False), False):
yield m
barrier = connection.openflowdef.ofp_msg.new()
barrier.header.type = connection.openflowdef.OFPT_BARRIER_REQUEST
for m in container.waitForSend(self.formatrequest(barrier, connection)):
yield m
barrierreply = self.replymatcher(barrier, connection)
yield (barrierreply,)
for m in container.withCallback(batchprocess(), callback, conndown, *replymatchers.keys()):
yield m
container.openflow_reply = replymessages
container.openflow_replydict = dict((replymatchers[k],v) for k,v in replydict.items())
if firsterror[0] and raiseonerror:
raise OpenflowErrorResultException('One or more error message is returned from a batch process, the first is: '
+ repr(dump(firsterror[0])))
def __init__(self,
errormsg,
prompt='An error message is returned: ',
result=None):
if result is None:
result = errormsg
Exception.__init__(self, prompt + repr(dump(errormsg)))
self.result = result
def main(self):
def _watch(w):
for m in w.wait(self.apiroutine):
yield m
print('Watcher returns:', dump(self.apiroutine.retvalue))
def _watchall(watchers):
for w in watchers:
if w is not None:
self.apiroutine.subroutine(_watch(w))
self.apiroutine.subroutine(self.watcher(), False, daemon = True)
up = ZooKeeperSessionStateChanged.createMatcher(ZooKeeperSessionStateChanged.CREATED, self.client)
yield (up,)
print('Connection is up: %r' % (self.client.currentserver,))
for m in self.client.requests([zk.create(b'/vlcptest', b'test'),
zk.getdata(b'/vlcptest', True)], self.apiroutine):
yield m
print(self.apiroutine.retvalue)
pprint(dump(self.apiroutine.retvalue[0]))
_watchall(self.apiroutine.retvalue[3])
for m in self.apiroutine.waitWithTimeout(0.2):
yield m
for m in self.client.requests([zk.delete(b'/vlcptest'),
zk.getdata(b'/vlcptest', watch = True)], self.apiroutine):
yield m
print(self.apiroutine.retvalue)
pprint(dump(self.apiroutine.retvalue[0]))
_watchall(self.apiroutine.retvalue[3])
for m in self.client.requests([zk.multi(
zk.multi_create(b'/vlcptest2', b'test'),
zk.multi_create(b'/vlcptest2/subtest', 'test2')
),
zk.getchildren2(b'/vlcptest2', True)], self.apiroutine):
yield m
print(self.apiroutine.retvalue)
pprint(dump(self.apiroutine.retvalue[0]))
_watchall(self.apiroutine.retvalue[3])
for m in self.client.requests([zk.multi(
zk.multi_delete(b'/vlcptest2/subtest'),
zk.multi_delete(b'/vlcptest2')),
zk.getchildren2(b'/vlcptest2', True)], self.apiroutine):
yield m
print(self.apiroutine.retvalue)
pprint(dump(self.apiroutine.retvalue[0]))
_watchall(self.apiroutine.retvalue[3])
def testDefs14(self):
for k in dir(openflow14):
attr = getattr(openflow14, k)
if isinstance(attr, nstruct) and not attr in self.exclude and not k.startswith('ofp_oxm_') and not k.startswith('nxm_') and not hasattr(ethernet, k):
if not attr.subclasses:
if k != repr(attr) and hasattr(openflow14, repr(attr)) and getattr(openflow14, repr(attr)) is attr:
# An alias
print(k, 'is alias of', repr(attr))
continue
self.assertEqual(k, repr(attr), k + ' has different name: ' + repr(attr))
print(k, repr(attr))
obj = attr.new()
s = obj._tobytes()
try:
r = attr.parse(s)
except Exception:
self.assertTrue(False, repr(attr) + ' failed to parse')
self.assertTrue(r is not None, repr(attr) + ' failed to parse')
obj2, size = r
self.assertEqual(size, len(s), repr(attr) + ' failed to parse')
self.assertEqual(dump(obj), dump(obj2), repr(attr) + ' changed after parsing')
def testEmbeddedMsg(self):
obj = openflow14.ofp_bundle_add_msg(message=openflow14.ofp_packet_out(data=b'abc'))
s = obj._tobytes()
# No padding
self.assertEqual(len(s), 43)
self.assertEqual(s[-3:], b'abc')
r = openflow14.ofp_msg.parse(s)
self.assertTrue(r is not None, 'failed to parse')
obj2, size = r
self.assertEqual(size, len(s), 'failed to parse')
self.assertEqual(dump(obj), dump(obj2), 'changed after parsing')
obj = openflow14.ofp_bundle_add_msg(message=openflow14.ofp_packet_out(data=b'abc'),
properties=[openflow14.ofp_bundle_prop_experimenter()])
s = obj._tobytes()
# auto padding
self.assertEqual(len(s), 64)
r = openflow14.ofp_msg.parse(s)
self.assertTrue(r is not None, 'failed to parse')
obj2, size = r
self.assertEqual(size, len(s), 'failed to parse')
self.assertEqual(dump(obj), dump(obj2), 'changed after parsing')
obj = openflow14.ofp_requestforward(request=openflow14.ofp_packet_out(data=b'abc'))
s = obj._tobytes()
# No padding
self.assertEqual(len(s), 35)
self.assertEqual(s[-3:], b'abc')
r = openflow14.ofp_msg.parse(s)
self.assertTrue(r is not None, 'failed to parse')
obj2, size = r
self.assertEqual(size, len(s), 'failed to parse')
self.assertEqual(dump(obj), dump(obj2), 'changed after parsing')
def main(self):
self.apiroutine.subroutine(self.watcher(), False, daemon = True)
up = ZooKeeperConnectionStateEvent.createMatcher(ZooKeeperConnectionStateEvent.UP, self.client)
notconn = ZooKeeperConnectionStateEvent.createMatcher(ZooKeeperConnectionStateEvent.NOTCONNECTED, self.client)
yield (up, notconn)
if self.apiroutine.matcher is notconn:
print('Not connected')
return
else:
print('Connection is up: %r' % (self.client,))
# Handshake
for m in self.protocol.handshake(self.client, zk.ConnectRequest(
timeOut = int(self.sessiontimeout * 1000),
passwd = b'\x00' * 16, # Why is it necessary...
), self.apiroutine, []):
yield m
for m in self.protocol.requests(self.client, [zk.create(b'/vlcptest', b'test'),
zk.getdata(b'/vlcptest', True)], self.apiroutine):
yield m
pprint(dump(self.apiroutine.retvalue[0]))
for m in self.apiroutine.waitWithTimeout(0.2):
yield m
for m in self.protocol.requests(self.client, [zk.delete(b'/vlcptest'),
zk.getdata(b'/vlcptest', watch = True)], self.apiroutine):
yield m
pprint(dump(self.apiroutine.retvalue[0]))
for m in self.protocol.requests(self.client, [zk.multi(
zk.multi_create(b'/vlcptest2', b'test'),
zk.multi_create(b'/vlcptest2/subtest', 'test2')
),
zk.getchildren2(b'/vlcptest2', True)], self.apiroutine):
yield m
pprint(dump(self.apiroutine.retvalue[0]))
for m in self.protocol.requests(self.client, [zk.multi(
zk.multi_delete(b'/vlcptest2/subtest'),
zk.multi_delete(b'/vlcptest2')),
zk.getchildren2(b'/vlcptest2', True)], self.apiroutine):
yield m
pprint(dump(self.apiroutine.retvalue[0]))
def batch(self, requests, connection, container, raiseonerror=True):
"""
Send multiple requests, return when all the requests are done. Requests can have no responses.
`container.openflow_reply` is the list of messages in receiving order. `container.openflow_replydict`
is the dictionary `{request:reply}`. The attributes are set even if an OpenflowErrorResultException is raised.
"""
for r in requests:
self.assignxid(r, connection)
replymatchers = dict(
(self.replymatcher(r, connection), r) for r in requests)
replydict = {}
replymessages = []
conndown = self.statematcher(connection)
firsterror = [None]
def callback(event, matcher):
if matcher is conndown:
raise OpenflowProtocolException(
'Connection is down before reply received')
msg = event.message
if event.iserror and not firsterror[0]:
firsterror[0] = msg
replydict.setdefault(matcher, []).append(msg)
replymessages.append(msg)
def batchprocess():
for r in requests:
for m in connection.write(
self.formatrequest(r, connection, False), False):
yield m
barrier = connection.openflowdef.ofp_msg.new()
barrier.header.type = connection.openflowdef.OFPT_BARRIER_REQUEST
for m in container.waitForSend(
self.formatrequest(barrier, connection)):
yield m
barrierreply = self.replymatcher(barrier, connection)
yield (barrierreply, )
for m in container.withCallback(batchprocess(), callback, conndown,
*replymatchers.keys()):
yield m
container.openflow_reply = replymessages
container.openflow_replydict = dict(
(replymatchers[k], v) for k, v in replydict.items())
if firsterror[0] and raiseonerror:
raise OpenflowErrorResultException(
'One or more error message is returned from a batch process, the first is: '
+ repr(dump(firsterror[0])))
def querywithreply(self,
request,
connection,
container,
raiseonerror=True):
for m in connection.write(self.formatrequest(request, connection),
False):
yield m
reply = self.replymatcher(request, connection)
conndown = self.statematcher(connection)
yield (reply, conndown)
if container.matcher is conndown:
raise OpenflowProtocolException(
'Connection is down before reply received')
container.openflow_reply = container.event.message
if container.event.iserror:
raise OpenflowErrorResultException(
'An error message is returned: ' +
repr(dump(container.event.message)))
def querywithreply(self,
request,
connection,
container,
raiseonerror=True):
"""
Send an OpenFlow normal request, wait for the response of this request. The request must have
exactly one response. The reply is stored to `container.openflow_reply`
"""
for m in connection.write(self.formatrequest(request, connection),
False):
yield m
reply = self.replymatcher(request, connection)
conndown = self.statematcher(connection)
yield (reply, conndown)
if container.matcher is conndown:
raise OpenflowProtocolException(
'Connection is down before reply received')
container.openflow_reply = container.event.message
if container.event.iserror:
raise OpenflowErrorResultException(
'An error message is returned: ' +
repr(dump(container.event.message)))
def querymultipart(self, request, connection, container, raiseonerror = True):
for m in connection.write(self.formatrequest(request, connection), False):
yield m
reply = self.replymatcher(request, connection)
conndown = self.statematcher(connection)
messages = []
while True:
yield (reply, conndown)
if container.matcher is conndown:
raise OpenflowProtocolException('Connection is down before reply received')
msg = container.event.message
messages.append(msg)
if msg.header.type == common.OFPT_ERROR or not (msg.flags & common.OFPSF_REPLY_MORE):
if msg.header.type == common.OFPT_ERROR and raiseonerror:
container.openflow_reply = messages
raise OpenflowErrorResultException('An error message is returned: ' + repr(dump(msg)))
break
container.openflow_reply = messages
def querywithreply(self, request, connection, container, raiseonerror = True):
for m in connection.write(self.formatrequest(request, connection), False):
yield m
reply = self.replymatcher(request, connection)
conndown = self.statematcher(connection)
yield (reply, conndown)
if container.matcher is conndown:
raise OpenflowProtocolException('Connection is down before reply received')
container.openflow_reply = container.event.message
if container.event.iserror:
raise OpenflowErrorResultException('An error message is returned: ' + repr(dump(container.event.message)))
verbose = None
length = args.length
if length <= 14:
length = 14
filters = [create_filter(f) for f in args.filter]
except Exception as exc:
print(str(exc), file=sys.stderr)
exit(2)
try:
total_packet = 0
filtered_packet = 0
while True:
data, addr = raw_socket.recvfrom(4096)
try:
p = parser(data, length)
pd = dump(p)
except Exception:
print('%s Unrecognized packet: %r' % (current_timestamp(), data[:28],))
else:
try:
total_packet += 1
if all(f(pd) for f in filters):
filtered_packet += 1
format_packet(pd, verbose, addr)
except Exception as exc:
print(str(exc), file=sys.stderr)
except (KeyboardInterrupt, SystemExit):
print()
print("%d/%d packet captured" % (filtered_packet, total_packet))
from __future__ import print_function
import socket
import protocoldef as d
from namedstruct import dump
if __name__ == '__main__':
s = socket.create_connection(('127.0.0.1', 9723))
try:
s.sendall(d.message[0].tobytes([
d.message10_sum_request(numbers=[1, 2, 3]),
d.message10_sum_request(numbers=list(range(0, 100))),
d.message10_sum_request(numbers=[i * i for i in range(0, 100)]),
]))
data = b''
while True:
data2 = s.recv(4096)
if not data2:
break
data += data2
r = d.message[3].parse(data)
if r:
messages, _ = r
break
print(dump(messages))
finally:
s.close()
def parse(self, connection, data, laststart):
events = []
start = 0
while True:
result = ZooKeeperReply.parse(data[start:])
if result is None:
break
reply, size = result
start += size
if not connection.zookeeper_handshake:
reply.zookeeper_type = CONNECT_PACKET
reply._autosubclass()
connection.zookeeper_handshake = True
events.append(
ZooKeeperHandshakeEvent(connection,
connection.connmark,
self,
message=reply))
else:
reply.zookeeper_type = HEADER_PACKET
reply._autosubclass()
if reply.zxid > 0:
connection.zookeeper_lastzxid = reply.zxid
if reply.xid >= 0:
xid = reply.xid
if xid not in connection.zookeeper_requests:
raise ZooKeeperProtocolException(
'xid does not match: receive %r' % (reply.xid, ))
request_type = connection.zookeeper_requests.pop(xid)
reply.zookeeper_request_type = request_type
reply._autosubclass()
if reply.xid == WATCHER_EVENT_XID:
connection.zookeeper_last_watch_zxid = connection.zookeeper_lastzxid
reply.last_zxid = connection.zookeeper_lastzxid
if reply.state != ZOO_SYNC_CONNECTED_STATE:
self._logger.warning(
"Receive abnormal watch event: %s",
dumps(dump(reply, tostr=True)))
events.append(
ZooKeeperWatcherEvent(
connection,
connection.connmark,
self,
reply.type,
reply.state,
b'' if reply.path is None else reply.path,
message=reply))
else:
events.append(
ZooKeeperResponseEvent(connection,
connection.connmark,
self,
reply.xid,
message=reply))
if laststart == len(data):
# Remote write close
events.append(
ConnectionWriteEvent(connection,
connection.connmark,
data=b'',
EOF=True))
return (events, len(data) - start)
def _watch(w):
for m in w.wait(self.apiroutine):
yield m
print('Watcher returns:', dump(self.apiroutine.retvalue))
def watcher(self):
watcher = ZooKeeperWatcherEvent.createMatcher()
while True:
yield (watcher,)
print('WatcherEvent: %r' % (dump(self.apiroutine.event.message),))
def querywithreply(self, request, connection, container, raiseonerror = True):
"""
Send an OpenFlow normal request, wait for the response of this request. The request must have
exactly one response. The reply is stored to `container.openflow_reply`
"""
for m in connection.write(self.formatrequest(request, connection), False):
yield m
reply = self.replymatcher(request, connection)
conndown = self.statematcher(connection)
yield (reply, conndown)
if container.matcher is conndown:
raise OpenflowProtocolException('Connection is down before reply received')
container.openflow_reply = container.event.message
if container.event.iserror:
raise OpenflowErrorResultException('An error message is returned: ' + repr(dump(container.event.message)))
def _watch(w):
for m in w.wait(self.apiroutine):
yield m
print('Watcher returns:', dump(self.apiroutine.retvalue))
from __future__ import print_function
import socket
import protocoldef as d
from namedstruct import dump
if __name__ == '__main__':
s = socket.create_connection(('127.0.0.1', 9723))
try:
s.sendall(d.message[0].tobytes([d.message10_sum_request(numbers = [1,2,3]),
d.message10_sum_request(numbers = list(range(0,100))),
d.message10_sum_request(numbers = [i*i for i in range(0,100)]),]))
data = b''
while True:
data2 = s.recv(4096)
if not data2:
break
data += data2
r = d.message[3].parse(data)
if r:
messages, _ = r
break
print(dump(messages))
finally:
s.close()
def watcher(self):
watcher = ZooKeeperWatcherEvent.createMatcher(connection = self.client)
while True:
yield (watcher,)
print('WatcherEvent: %r' % (dump(self.apiroutine.event.message),))
async def _connection_manage(self):
try:
failed = 0
self._last_zxid = last_zxid = 0
self._last_watch_zxid = last_watch_zxid = 0
session_id = 0
passwd = b'\x00' * 16
last_conn_time = None
while True:
self.currentserver = self.serverlist[self.nextptr]
np = self.nextptr + 1
if np >= len(self.serverlist):
np = 0
self.nextptr = np
conn = Client(self.currentserver, self.protocol,
self._container.scheduler, self.key,
self.certificate, self.ca_certs)
self.current_connection = conn
conn_up = ZooKeeperConnectionStateEvent.createMatcher(
ZooKeeperConnectionStateEvent.UP, conn)
conn_nc = ZooKeeperConnectionStateEvent.createMatcher(
ZooKeeperConnectionStateEvent.NOTCONNECTED, conn)
conn.start()
try:
_, m = await M_(conn_up, conn_nc)
if m is conn_nc:
self._logger.warning(
'Connect to %r failed, try next server',
self.currentserver)
if failed > 5:
# Wait for a small amount of time to prevent a busy loop
# Socket may be rejected, it may fail very quick
await self._container.wait_with_timeout(
min((failed - 5) * 0.1, 1.0))
failed += 1
continue
try:
# Handshake
set_watches = []
if self.session_state == ZooKeeperSessionStateChanged.DISCONNECTED:
await self._container.wait_for_send(
ZooKeeperRestoreWatches(
self,
self.session_id,
True,
restore_watches=(set(), set(), set())))
ev = await ZooKeeperRestoreWatches.createMatcher(
self)
data_watches, exists_watches, child_watches = \
ev.restore_watches
if data_watches or exists_watches or child_watches:
restore_watch_zxid = last_watch_zxid
if restore_watch_zxid > 1:
restore_watch_zxid = restore_watch_zxid - 1
current_set_watches = zk.SetWatches(
relativeZxid=restore_watch_zxid)
current_length = 0
for d, e, c in izip_longest(
data_watches, exists_watches,
child_watches):
if d is not None:
current_set_watches.dataWatches.append(
d)
current_length += 4 + len(d)
if e is not None:
current_set_watches.existWatches.append(
e)
current_length += 4 + len(e)
if c is not None:
current_set_watches.childWatches.append(
c)
current_length += 4 + len(c)
if current_length > _MAX_SETWATCHES_SIZE:
# Split set_watches
set_watches.append(current_set_watches)
current_set_watches = zk.SetWatches(
relativeZxid=restore_watch_zxid)
current_length = 0
if current_set_watches.dataWatches or current_set_watches.existWatches \
or current_set_watches.childWatches:
set_watches.append(current_set_watches)
auth_list = list(self.auth_set)
extra_requests = [
zk.AuthPacket(scheme=a[0], auth=a[1])
for a in auth_list
] + set_watches
timeout, handshake_result = await self._container.execute_with_timeout(
10,
self.protocol.handshake(
conn,
zk.ConnectRequest(
lastZxidSeen=last_zxid,
timeOut=int(self.sessiontimeout * 1000.0),
sessionId=session_id,
passwd=passwd,
readOnly=self.readonly), self._container,
extra_requests))
if timeout:
raise IOError
except ZooKeeperSessionExpiredException:
self._logger.warning('Session expired.')
# Session expired
self.session_state = ZooKeeperSessionStateChanged.EXPIRED
await self._container.wait_for_send(
ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.EXPIRED, self,
session_id))
if self.restart_session:
failed = 0
last_zxid = 0
session_id = 0
passwd = b'\x00' * 16
last_conn_time = None
continue
else:
break
except Exception:
self._logger.warning(
'Handshake failed to %r, try next server',
self.currentserver)
if failed > 5:
# There is a bug ZOOKEEPER-1159 that ZooKeeper server does not respond
# for session expiration, but directly close the connection.
# This is a workaround: we store the time that we disconnected from the server,
# if we have exceeded the session expiration time, we declare the session is expired
if last_conn_time is not None and last_conn_time + self.sessiontimeout * 2 < time(
):
self._logger.warning(
'Session expired detected from client time.'
)
# Session expired
self.session_state = ZooKeeperSessionStateChanged.EXPIRED
await self._container.wait_for_send(
ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.EXPIRED,
self, session_id))
if self.restart_session:
failed = 0
last_zxid = 0
session_id = 0
passwd = b'\x00' * 16
last_conn_time = None
continue
else:
break
else:
# Wait for a small amount of time to prevent a busy loop
await self._container.wait_with_timeout(
min((failed - 5) * 0.1, 1.0))
failed += 1
else:
failed = 0
conn_resp, auth_resp = handshake_result
if conn_resp.timeOut <= 0:
# Session expired
# Currently should not happen because handshake() should raise an exception
self._logger.warning(
'Session expired detected from handshake packet'
)
self.session_state = ZooKeeperSessionStateChanged.EXPIRED
await self._container.wait_for_send(
ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.EXPIRED, self,
session_id))
if self.restart_session:
failed = 0
last_zxid = 0
last_conn_time = None
session_id = 0
passwd = b'\x00' * 16
continue
else:
break
else:
session_id = conn_resp.sessionId
passwd = conn_resp.passwd
# Authentication result check
auth_failed = any(a.err == zk.ZOO_ERR_AUTHFAILED
for a in auth_resp)
if auth_failed:
self._logger.warning(
'ZooKeeper authentication failed for following auth: %r',
[
a
for a, r in zip(auth_list, auth_resp)
if r.err == zk.ZOO_ERR_AUTHFAILED
])
self.session_state = ZooKeeperSessionStateChanged.AUTHFAILED
await self._container.wait_for_send(
ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.
AUTHFAILED, self, session_id))
# Not retrying
break
else:
# Check other failures
failed_results = [[
r, a
] for r, a in zip(extra_requests, auth_resp)
if a.err != zk.ZOO_ERR_OK]
if failed_results:
# What's wrong? log them and ignored
self._logger.warning(
'Some handshake packets have error result:\n%s',
json.dumps(dump(failed_results,
tostr=True),
indent=2))
self.session_readonly = getattr(
conn_resp, 'readOnly', False)
self.session_id = session_id
if self.session_state == ZooKeeperSessionStateChanged.EXPIRED:
await self._container.wait_for_send(
ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.
CREATED, self, session_id))
else:
await self._container.wait_for_send(
ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.
RECONNECTED, self, session_id))
self.session_state = ZooKeeperSessionStateChanged.CREATED
if conn.connected:
conn_down = ZooKeeperConnectionStateEvent.createMatcher(
ZooKeeperConnectionStateEvent.DOWN, conn,
conn.connmark)
auth_failed = ZooKeeperResponseEvent.createMatcher(
zk.AUTH_XID,
conn,
conn.connmark,
_ismatch=lambda x: x.message.err ==
ZOO_ERR_AUTHFAILED)
while True:
rebalancetime = self.rebalancetime
if rebalancetime is not None:
rebalancetime += random() * 60
timeout, ev, m = await self._container.wait_with_timeout(
rebalancetime, conn_down, auth_failed)
if timeout:
# Rebalance
if conn.zookeeper_requests:
# There are still requests not processed, wait longer
for _ in range(0, 3):
longer_time = random() * 10
timeout, ev, m = await self._container.wait_with_timeout(
longer_time, conn_down,
auth_failed)
if not timeout:
# Connection is down, or auth failed
break
if not conn.zookeeper_requests:
break
else:
# There is still requests, skip for this time
continue
# Rebalance to a random server
if timeout:
self.nextptr = randrange(
len(self.serverlist))
break
if m is auth_failed:
self._logger.warning(
'ZooKeeper authentication failed, shutdown the connection'
)
self.session_state = ZooKeeperSessionStateChanged.AUTHFAILED
await self._container.wait_for_send(
ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.
AUTHFAILED, self, session_id))
# Not retrying
break
else:
# Connection is down, try other servers
if not timeout:
self._logger.warning(
'Connection lost to %r, try next server',
self.currentserver)
else:
self._logger.info(
'Rebalance to next server')
self._last_zxid = last_zxid = conn.zookeeper_lastzxid
self._last_watch_zxid = last_watch_zxid = conn.zookeeper_last_watch_zxid
last_conn_time = time()
self.session_state = ZooKeeperSessionStateChanged.DISCONNECTED
await self._container.wait_for_send(
ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.
DISCONNECTED, self, session_id))
finally:
conn.subroutine(conn.shutdown(True), False)
self.current_connection = None
finally:
self._shutdown = True
if self.session_state != ZooKeeperSessionStateChanged.EXPIRED and self.session_state != ZooKeeperSessionStateChanged.AUTHFAILED:
self.session_state = ZooKeeperSessionStateChanged.EXPIRED
self._container.scheduler.emergesend(
ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.EXPIRED, self,
session_id))
