def mdp_request(socket, service, msg, timeout=None):
"""Synchronous MDP request.
This function sends a request to the given service and
waits for a reply.
If timeout is set and no reply received in the given time
the function will return `None`.
:param socket: zmq REQ socket to use.
:type socket: zmq.Socket
:param service: service id to send the msg to.
:type service: str
:param msg: list of message parts to send.
:type msg: list of str
:param timeout: time to wait for answer in seconds.
:type timeout: float
:rtype list of str:
"""
if not timeout or timeout < 0.0:
timeout = None
to_send = [PROTO_VERSION, service]
to_send.extend(msg)
socket.send_multipart(to_send)
ret = None
rlist, _, _ = select([socket], [], [], timeout)
if rlist and rlist[0] == socket:
ret = socket.recv_multipart()
ret.pop(0) # remove service from reply
return ret
def _poll(self,sockets,timeout=None):
# Don't bother trampolining if there's data available immediately.
# This also avoids calling into the eventlet hub with a timeout of
# zero, which doesn't work right (it still switches the greenthread)
(r,_,_) = zmq_poll.select(sockets,[],[],timeout=0)
if r:
return r
if timeout == 0:
return []
# Looks like we'll have to block :-(
ready = []
threads = []
res = Event()
for sock in sockets:
threads.append(eventlet.spawn(self._do_poll,sock,ready,res,timeout))
self.poll_threads.append((res,threads))
try:
res.wait()
finally:
self.poll_threads.remove((res,threads))
for t in threads:
t.kill()
try:
t.wait()
except GreenletExit:
pass
return ready
def mdp_request(socket, service, msg, timeout=None):
"""Synchronous MDP request.
This function sends a request to the given service and
waits for a reply.
If timeout is set and no reply received in the given time
the function will return `None`.
:param socket: zmq REQ socket to use.
:type socket: zmq.Socket
:param service: service id to send the msg to.
:type service: str
:param msg: list of message parts to send.
:type msg: list of str
:param timeout: time to wait for answer in seconds.
:type timeout: float
:rtype list of str:
"""
if not timeout or timeout < 0.0:
timeout = None
if type(msg) in (str, unicode):
msg = [msg]
to_send = [PROTO_VERSION, service]
to_send.extend(msg)
socket.send_multipart(to_send)
ret = None
rlist, _, _ = select([socket], [], [], timeout)
if rlist and rlist[0] == socket:
ret = socket.recv_multipart()
ret.pop(0) # remove service from reply
return ret
def _poll(self, sockets, timeout=None):
# Don't bother trampolining if there's data available immediately.
# This also avoids calling into the eventlet hub with a timeout of
# zero, which doesn't work right (it still switches the greenthread)
(r, _, _) = zmq_poll.select(sockets, [], [], timeout=0)
if r:
return r
if timeout == 0:
return []
# Looks like we'll have to block :-(
ready = []
threads = []
res = Event()
for sock in sockets:
threads.append(
eventlet.spawn(self._do_poll, sock, ready, res, timeout))
self.poll_threads.append((res, threads))
try:
res.wait()
finally:
self.poll_threads.remove((res, threads))
for t in threads:
t.kill()
try:
t.wait()
except GreenletExit:
pass
return ready
def _poll(self,sockets,timeout=None):
# If there's anything available non-blockingly, just use it.
(ready,_,error) = zmq_poll.select(sockets,[],sockets,timeout=0)
if ready:
return ready
if error:
return []
if timeout == 0:
return []
# Spawn a greenthread to poll-recv from each socket.
ready = []
threads = []
res = gevent.event.Event()
for sock in sockets:
threads.append(gevent.spawn(self._do_poll,sock,ready,res,timeout))
self.poll_threads.append((res,threads))
# Wait for one of them to return, or for an interrupt.
try:
res.wait()
finally:
gevent.killall(threads)
gevent.joinall(threads)
return ready
def _poll(self, sockets, timeout=None):
# If there's anything available non-blockingly, just use it.
(ready, _, error) = zmq_poll.select(sockets, [], sockets, timeout=0)
if ready:
return ready
if error:
return []
if timeout == 0:
return []
# Spawn a greenthread to poll-recv from each socket.
ready = []
threads = []
res = gevent.event.Event()
for sock in sockets:
threads.append(
gevent.spawn(self._do_poll, sock, ready, res, timeout))
self.poll_threads.append((res, threads))
# Wait for one of them to return, or for an interrupt.
try:
res.wait()
finally:
gevent.killall(threads)
gevent.joinall(threads)
return ready
def _loop():
# Make a copy of the connections registry. Copy is atomic due to the GIL.
remote_node_addresses = set(self._connections.keys())
# Add any connections that we are not monitoring
for remote_node_address in remote_node_addresses:
if remote_node_address not in hb_socks_registry:
hb_socket = _create_socket(self._ctx, zmq.XREQ)
# The identity of the heartbeat socket should be constant for any given
# local-node and remote-node
hb_socket.identity = self.address + "->" + remote_node_address
# Stash away the remote endpoint for each socket (this is somehow not exposed
# by the socket itself)
endpoints[hb_socket] = remote_node_address
hb_socket.connect(remote_node_address)
hb_socks_registry[remote_node_address] = hb_socket
# Remove heartbeat sockets that we no longer need
for remote_node_address in hb_socks_registry.keys():
if remote_node_address not in remote_node_addresses:
log.debug("Removing disconnected node %s from heartbeat registry.", remote_node_address)
sock = hb_socks_registry.pop(remote_node_address)
del endpoints[sock]
sock.close()
remote_node_count = len(hb_socks_registry)
log.debug("Pinging %d remote nodes in registry", remote_node_count)
# If there are no sockets to watch, skip the pinging bit
if not remote_node_count:
# Wait a bit before retrying
time.sleep(self._timeout)
return
# Get a list of the actual sockets
hb_socks = hb_socks_registry.values()
# Send a ping to all sockets
for sock in hb_socks:
sock.send_multipart(message.can_ctr(ctr_ping))
# Wait at most <timeout> seconds for a reply.
time.sleep(self._timeout)
# Get all sockets that responded (wait a bit extra if there are no responses at all)
resp, _, _ = zpoll.select(hb_socks, [], [], timeout=10)
# Get a list of nodes which did not respond in time
failures = set(hb_socks).difference(resp)
for sock in resp:
# TODO: Add some handling for the replies. At the moment we just throw away them away.
sock.recv_multipart(zmq.NOBLOCK)
num_failures = len(failures)
if num_failures > 0:
log.warn("%d nodes failed to respond", num_failures)
dead_node_addresses = [endpoints[failed_hb_sock] for failed_hb_sock in failures]
# Send a message to the main execution loop that it should kill all pending
# requests on the affected channels
msg_disconnect = message.can_disconnect_channels(dead_node_addresses, "TIMEOUT")
msg_disconnect = message.can_msg(_msg_disconnect_channels, msg_disconnect)
msg_disconnect = message.can_ctr(ctr_data, msg_disconnect)
exec_loop_sock.send_multipart([self.address] + msg_disconnect, copy=False)