def test_message_order(self):
"""test that we get messages in the order we expect"""
queue = PriorityQueue()
messages = [
message_format(
ident=None,
control={"priority" : 0, "expected-order" : 0},
body=None
),
message_format(
ident=None,
control={"priority" : 5, "expected-order" : 2},
body=None
),
message_format(
ident=None,
control={"priority" : 3, "expected-order" : 1},
body=None
),
]
self.assertEqual(len(queue), 0)
for message in messages:
queue.append((message.control, message.body, ))
self.assertEqual(len(queue), len(messages))
for order in range(len(messages)):
retrieved_message, _retrieved_data = queue.popleft()
self.assertEqual(
order, retrieved_message["expected-order"]
)
self.assertEqual(len(queue), 0)
self.assertRaises(IndexError, queue.popleft)
def __init__(
self,
context,
server_node_name,
server_address,
client_tag,
client_address,
deliverator,
connect_messages=list(),
):
Greenlet.__init__(self)
self._log = logging.getLogger("ResilientClient-%s" % (server_address,))
self._context = context
self._server_node_name = server_node_name
self._server_address = server_address
self._client_tag = client_tag
self._client_address = client_address
self._deliverator = deliverator
self._send_queue = gevent.queue.Queue()
# prime the send queue with messages to be sent as soon
# as we connect
for connect_message in connect_messages:
if not "message-id" in connect_message:
connect_message["message-id"] = uuid.uuid1().hex
message = message_format(ident=None, control=connect_message, body=None)
self._send_queue.put(message)
self._req_socket = None
self.connected = False
def queue_message_for_send(self, message_control, data=None):
"""
message control must be a dict
If the caller includes a 'message-id' key we will use it,
otherwise, we will supply one.
return: a gevent queue (zero size Queue) the the reply will
be deliverd.
"""
if not self.connected:
raise ResilientClientError(
"queue_message_for_send while not connected %s" % (
message_control,
)
)
if not "message-id" in message_control:
message_control["message-id"] = uuid.uuid1().hex
message_id = message_control["message-id"]
delivery_channel = self._deliverator.add_request(message_id)
message = message_format(
ident=None, control=message_control, body=data
)
self._send_queue.put(message)
return delivery_channel
def _handle_status_disconnected(self):
elapsed_time = time.time() - self._status_time
if elapsed_time < _handshake_retry_interval:
return
assert self._dealer_socket is None
self._dealer_socket = self._context.socket(zmq.XREQ)
self._dealer_socket.setsockopt(zmq.LINGER, 1000)
self._log.debug("connecting to server")
self._dealer_socket.connect(self._server_address)
self._pollster.register_read(
self._dealer_socket, self._pollster_callback
)
message = {
"message-type" : "resilient-server-handshake",
"message-id" : uuid.uuid1().hex,
"client-tag" : self._client_tag,
"client-address" : self._client_address,
}
message = message_format(ident=None, control=message, body=None)
self._pending_message = message
self._pending_message_start_time = time.time()
self._status = _status_handshaking
self._status_time = time.time()
self._send_message(message)
def queue_message_for_send(self, message_control, data=None):
"""
message_control
a dictionary, to be sent as JSON
data (optional)
binary data (possibly a sequence of strings) to be sent in
the message
queue a message for send (unless we can send it immediately)
if message_control does not contain a message-id, we will supply one.
"""
if not "message-id" in message_control:
message_control["message-id"] = uuid.uuid1().hex
message = message_format(
ident=None, control=message_control, body=data
)
if self._status is _status_connected and self._pending_message is None:
self._pending_message = message
self._pending_message_start_time = time.time()
self._send_message(message)
else:
self._send_queue.append(message)
def queue_message_for_send(self, message_control, data=None):
message_ident = message_control.pop("router-ident")
self._send_queue.append(
message_format(
ident=message_ident, control=message_control, body=data
)
)
def _complete_handoff(self, message, data, completion_channel):
# hand off the message,
if "handoff-node-name" in message:
assert message["handoff-node-name"] is None, message
message["handoff-node-name"] = self._dest_node_name
data_writer_greenlets = [
# queue a copy of the message, so each gets a different message-id
gevent.spawn(
self._hand_off_to_one_data_writer, client, message.copy(), data
) \
for client in self._backup_clients
]
# get all replies from the actual clients
gevent.joinall(data_writer_greenlets, timeout=_data_writer_timeout)
all_ready = all([g.ready() for g in data_writer_greenlets])
if not all_ready:
self._log.error("Not all data writer greenlets finished")
assert all_ready
data_writer_replies = [g.value for g in data_writer_greenlets]
# if any data writer has failed, we have failed
for data_writer_reply in data_writer_replies:
if data_writer_reply["result"] != "success":
reply = {
"message-type": "handoff-failure",
"result": "handoff-failure",
"error-message": data_writer_reply["error-message"],
}
message = message_format(ident=None, control=reply, body=None)
completion_channel.put((
message.control,
message.body,
))
return
# if we made it here all the handoffs succeeded,
# notify the sender that we are ok so far
# by sending one of the data_writer_replies
reply = data_writer_replies[0]
message = message_format(ident=None, control=reply, body=None)
completion_channel.put((
message.control,
message.body,
))
def queue_message_for_broadcast(self, message_control, data=None):
"""
queue a message for send, but do not create a delivery channel:
we are not expecting a reply
"""
if not "message-id" in message_control:
message_control["message-id"] = uuid.uuid1().hex
message = message_format(ident=None, control=message_control, body=data)
self._send_queue.put(message)
def _complete_handoff(self, message, data, completion_channel):
# hand off the message,
if "handoff-node-name" in message:
assert message["handoff-node-name"] is None, message
message["handoff-node-name"] = self._dest_node_name
data_writer_greenlets = [
# queue a copy of the message, so each gets a different message-id
gevent.spawn(
self._hand_off_to_one_data_writer, client, message.copy(), data
) \
for client in self._backup_clients
]
# get all replies from the actual clients
gevent.joinall(data_writer_greenlets, timeout=_data_writer_timeout)
all_ready = all([g.ready() for g in data_writer_greenlets])
if not all_ready:
self._log.error("Not all data writer greenlets finished")
assert all_ready
data_writer_replies = [g.value for g in data_writer_greenlets]
# if any data writer has failed, we have failed
for data_writer_reply in data_writer_replies:
if data_writer_reply["result"] != "success":
reply = {
"message-type" : "handoff-failure",
"result" : "handoff-failure",
"error-message" : data_writer_reply["error-message"],
}
message = message_format(ident=None, control=reply, body=None)
completion_channel.put((message.control, message.body, ))
return
# if we made it here all the handoffs succeeded,
# notify the sender that we are ok so far
# by sending one of the data_writer_replies
reply = data_writer_replies[0]
message = message_format(ident=None, control=reply, body=None)
completion_channel.put((message.control, message.body, ))
def test_single_entry(self):
"""test adding and removing a single entry"""
queue = PriorityQueue()
message = message_format(
ident=None, control={"priority" : 0}, body=None
)
self.assertEqual(len(queue), 0)
queue.append((message.control, message.body, ))
self.assertEqual(len(queue), 1)
retrieved_message, _retrieved_body = queue.popleft()
self.assertEqual(retrieved_message, message.control)
self.assertEqual(len(queue), 0)
self.assertRaises(IndexError, queue.popleft)
def queue_message_for_broadcast(self, message_control, data=None):
"""
queue a message for send, but do not create a delivery channel:
we are not expecting a reply
"""
if not "message-id" in message_control:
message_control["message-id"] = uuid.uuid1().hex
message = message_format(
ident=None, control=message_control, body=data
)
self._send_queue.put(message)
def test_message_order(self):
"""test that we get messages in the order we expect"""
queue = PriorityQueue()
messages = [
message_format(ident=None,
control={
"priority": 0,
"expected-order": 0
},
body=None),
message_format(ident=None,
control={
"priority": 5,
"expected-order": 2
},
body=None),
message_format(ident=None,
control={
"priority": 3,
"expected-order": 1
},
body=None),
]
self.assertEqual(len(queue), 0)
for message in messages:
queue.append((
message.control,
message.body,
))
self.assertEqual(len(queue), len(messages))
for order in range(len(messages)):
retrieved_message, _retrieved_data = queue.popleft()
self.assertEqual(order, retrieved_message["expected-order"])
self.assertEqual(len(queue), 0)
self.assertRaises(IndexError, queue.popleft)
def _receive_message(self):
control = self._rep_socket.recv_json()
body = []
while self._rep_socket.rcvmore:
body.append(self._rep_socket.recv())
# 2011-04-06 dougfort -- if someone is expecting a list and we only get
# one segment, they are going to have to deal with it.
if len(body) == 0:
body = None
elif len(body) == 1:
body = body[0]
return message_format(ident=None, control=control, body=body)
def _receive_message(self):
control = self._rep_socket.recv_json()
body = []
while self._rep_socket.rcvmore:
body.append(self._rep_socket.recv())
# 2011-04-06 dougfort -- if someone is expecting a list and we only get
# one segment, they are going to have to deal with it.
if len(body) == 0:
body = None
elif len(body) == 1:
body = body[0]
return message_format(ident=None, control=control, body=body)
def queue_message_for_send(self, message_control, data=None):
"""
message control must be a dict
If the caller includes a 'message-id' key we will use it,
otherwise, we will supply one.
return: a gevent queue (zero size Queue) the the reply will
be deliverd.
"""
if not "message-id" in message_control:
message_control["message-id"] = uuid.uuid1().hex
self._delivery_queues[message_control["message-id"]] = \
Queue(maxsize=None)
self._send_queue.put(
message_format(ident=None, control=message_control, body=data))
return self._delivery_queues[message_control["message-id"]]
def test_single_entry(self):
"""test adding and removing a single entry"""
queue = PriorityQueue()
message = message_format(ident=None,
control={"priority": 0},
body=None)
self.assertEqual(len(queue), 0)
queue.append((
message.control,
message.body,
))
self.assertEqual(len(queue), 1)
retrieved_message, _retrieved_body = queue.popleft()
self.assertEqual(retrieved_message, message.control)
self.assertEqual(len(queue), 0)
self.assertRaises(IndexError, queue.popleft)
def queue_message_for_send(self, message_control, data=None):
"""
message control must be a dict
If the caller includes a 'message-id' key we will use it,
otherwise, we will supply one.
return: a gevent queue (zero size Queue) the the reply will
be deliverd.
"""
if not "message-id" in message_control:
message_control["message-id"] = uuid.uuid1().hex
self._delivery_queues[message_control["message-id"]] = \
Queue(maxsize=None)
self._send_queue.put(
message_format(ident=None, control=message_control, body=data)
)
return self._delivery_queues[message_control["message-id"]]
def _run(self):
while True:
control = self._pull_socket.recv_json()
body = []
while self._pull_socket.rcvmore:
body.append(self._pull_socket.recv())
# 2011-04-06 dougfort -- if someone is expecting a list and we
# only get one segment, they are going to have to deal with it.
if len(body) == 0:
body = None
elif len(body) == 1:
body = body[0]
message = message_format(ident=None, control=control, body=body)
self._log.debug("received: %s" % (message.control, ))
self._deliverator.deliver_reply(message)
def _run(self):
while True:
control = self._pull_socket.recv_json()
body = []
while self._pull_socket.rcvmore:
body.append(self._pull_socket.recv())
# 2011-04-06 dougfort -- if someone is expecting a list and we
# only get one segment, they are going to have to deal with it.
if len(body) == 0:
body = None
elif len(body) == 1:
body = body[0]
message = message_format(ident=None, control=control, body=body)
self._log.debug("received: %s" % (message.control, ))
self._deliverator.deliver_reply(message)
def _deliver_failure_reply(self, message_to_send):
"""
deliver a failure reply to everyone waiting for this socket
"""
work_message = message_to_send
while work_message is not None:
reply = {
"message-type" : "ack-timeout-reply",
"message-id" : work_message.control["message-id"],
"result" : "ack timeout",
"error-message" : "timeout waiting ack: treating as disconnect"
}
message = message_format(ident=None, control=reply, body=None)
self._deliverator.deliver_reply(message)
try:
work_message = self._send_queue.get_nowait()
except gevent.queue.Empty:
work_message = None
def _receive_message(self):
try:
control = self._pull_socket.recv_json(zmq.NOBLOCK)
except zmq.ZMQError:
instance = sys.exc_info()[1]
if instance.errno == zmq.EAGAIN:
self._log.warn("socket would have blocked")
return None
raise
body = []
while self._pull_socket.rcvmore:
body.append(self._pull_socket.recv())
# 2011-04-06 dougfort -- if someone is expecting a list and we only get
# one segment, they are going to have to deal with it.
if len(body) == 0:
body = None
elif len(body) == 1:
body = body[0]
return message_format(ident=None, control=control, body=body)
def __init__(
self,
context,
server_node_name,
server_address,
client_tag,
client_address,
deliverator,
connect_messages=list()
):
Greenlet.__init__(self)
self._log = logging.getLogger("ResilientClient-%s" % (
server_address,
))
self._context = context
self._server_node_name = server_node_name
self._server_address = server_address
self._client_tag = client_tag
self._client_address = client_address
self._deliverator = deliverator
self._send_queue = gevent.queue.Queue()
# prime the send queue with messages to be sent as soon
# as we connect
for connect_message in connect_messages:
if not "message-id" in connect_message:
connect_message["message-id"] = uuid.uuid1().hex
message = message_format(
ident=None, control=connect_message, body=None
)
self._send_queue.put(message)
self._req_socket = None
self.connected = False
def queue_message_for_send(self, message_control, data=None):
self._send_queue.append(
message_format(ident=None, control=message_control, body=data))
class DealerClient(object):
"""
a class that manages a zeromq DEALER (aka XREQ) socket as a client
"""
def __init__(self, context, address, receive_queue):
self._log = logging.getLogger("DealerClient-%s" % (address, ))
self._dealer_socket = context.socket(zmq.XREQ)
self._dealer_socket.setsockopt(zmq.LINGER, 1000)
self._log.debug("connecting")
self._dealer_socket.connect(address)
self._send_queue = deque()
self._receive_queue = receive_queue
def register(self, pollster):
pollster.register_read_or_write(self._dealer_socket,
self._pollster_callback)
def unregister(self, pollster):
pollster.unregister(self._dealer_socket)
def close(self):
self._dealer_socket.close()
def queue_message_for_send(self, message_control, data=None):
self._send_queue.append(
message_format(ident=None, control=message_control, body=data))
def _pollster_callback(self, _active_socket, readable, writable):
# push our output first
while writable:
try:
message = self._send_queue.popleft()
except IndexError:
break
self._send_message(message)
# if we have input, read it and queue it
if readable:
message = self._receive_message()
# if we get None, that means the socket would have blocked
# go back and wait for more
if message is None:
return None
self._receive_queue.append((
message.control,
message.body,
))
def _send_message(self, message):
self._log.debug("sending message: %s" % (message.control, ))
# don't send a zero size body
if type(message.body) not in [
list,
tuple,
type(None),
]:
if len(message.body) == 0:
message = message._replace(body=None)
else:
message = message._replace(body=[
message.body,
])
if message.body is None:
self._dealer_socket.send_json(message.control)
else:
self._dealer_socket.send_json(message.control, zmq.SNDMORE)
for segment in message.body[:-1]:
self._dealer_socket.send(segment, zmq.SNDMORE)
self._dealer_socket.send(message.body[-1])
def _receive_message(self):
try:
control = self._dealer_socket.recv_json(zmq.NOBLOCK)
except zmq.ZMQError, instance:
if instance.errno == zmq.EAGAIN:
self._log.warn("socket would have blocked")
return None
raise
body = []
while self._dealer_socket.rcvmore:
body.append(self._dealer_socket.recv())
# 2011-04-06 dougfort -- if someone is expecting a list and we only get
# one segment, they are going to have to deal with it.
if len(body) == 0:
body = None
elif len(body) == 1:
body = body[0]
return message_format(ident=None, control=control, body=body)
def send_reply(self, message_control, data=None):
message = message_format(ident=None,
control=message_control,
body=data)
self._send_message(message)
def send_reply(self, message_control, data=None):
message = message_format(ident=None,
control=message_control,
body=data)
self._send_message(message)
class RouterServer(object):
"""
a class that manages a zeromq ROUTER (aka XREP) socket as a server
"""
def __init__(self, context, address, receive_queue):
self._log = logging.getLogger("RouterServer-%s" % (address, ))
# we need a valid path for IPC sockets
if address.startswith("ipc://"):
prepare_ipc_path(address)
self._router_socket = context.socket(zmq.XREP)
self._router_socket.setsockopt(zmq.LINGER, 1000)
self._log.debug("binding")
self._router_socket.bind(address)
self._send_queue = deque()
self._receive_queue = receive_queue
def register(self, pollster):
pollster.register_read_or_write(self._router_socket,
self._pollster_callback)
def unregister(self, pollster):
pollster.unregister(self._router_socket)
def close(self):
self._router_socket.close()
def queue_message_for_send(self, message_control, data=None):
message_ident = message_control.pop("router-ident")
self._send_queue.append(
message_format(ident=message_ident,
control=message_control,
body=data))
def _pollster_callback(self, _active_socket, readable, writable):
# push our output first, no point in taking on new work
# when we haven't sent our old stuff
while writable:
try:
message = self._send_queue.popleft()
except IndexError:
break
self._send_message(message)
# if we have input, read it and handle it
if readable:
message = self._receive_message()
# if we get None, that means the socket would have blocked
# go back and wait for more
if message is None:
return
# stuff the ident into the message, we'll want it back
# if the caller tries to send something
message.control["router-ident"] = message.ident
self._receive_queue.append((
message.control,
message.body,
))
def _send_message(self, message):
self._log.debug("sending message: %s" % (message.control, ))
self._router_socket.send(message.ident, zmq.SNDMORE)
# don't send a zero size body
if type(message.body) not in [
list,
tuple,
type(None),
]:
if len(message.body) == 0:
message = message._replace(body=None)
else:
message = message._replace(body=[
message.body,
])
if message.body is None:
self._router_socket.send_json(message.control)
else:
self._router_socket.send_json(message.control, zmq.SNDMORE)
for segment in message.body[:-1]:
self._router_socket.send(segment, zmq.SNDMORE)
self._router_socket.send(message.body[-1])
def _receive_message(self):
try:
ident = self._router_socket.recv(zmq.NOBLOCK)
except zmq.ZMQError, instance:
if instance.errno == zmq.EAGAIN:
self._log.warn("socket would have blocked")
return None
raise
assert self._router_socket.rcvmore, \
"Unexpected missing message control part."
control = self._router_socket.recv_json()
body = []
while self._router_socket.rcvmore:
body.append(self._router_socket.recv())
# 2011-04-06 dougfort -- if someone is expecting a list and we only get
# one segment, they are going to have to deal with it.
if len(body) == 0:
body = None
elif len(body) == 1:
body = body[0]
return message_format(ident=ident, control=control, body=body)
def queue_message_for_send(self, message_control, data=None):
message_ident = message_control.pop("router-ident")
self._send_queue.append(
message_format(ident=message_ident,
control=message_control,
body=data))
def queue_message_for_send(self, message_control, data=None):
self._send_queue.append(
message_format(ident=None, control=message_control, body=data)
)
