def test_message_order(self): """test that we get messages in the order we expect""" queue = PriorityQueue() messages = [ message_format( ident=None, control={"priority" : 0, "expected-order" : 0}, body=None ), message_format( ident=None, control={"priority" : 5, "expected-order" : 2}, body=None ), message_format( ident=None, control={"priority" : 3, "expected-order" : 1}, body=None ), ] self.assertEqual(len(queue), 0) for message in messages: queue.append((message.control, message.body, )) self.assertEqual(len(queue), len(messages)) for order in range(len(messages)): retrieved_message, _retrieved_data = queue.popleft() self.assertEqual( order, retrieved_message["expected-order"] ) self.assertEqual(len(queue), 0) self.assertRaises(IndexError, queue.popleft)
def __init__( self, context, server_node_name, server_address, client_tag, client_address, deliverator, connect_messages=list(), ): Greenlet.__init__(self) self._log = logging.getLogger("ResilientClient-%s" % (server_address,)) self._context = context self._server_node_name = server_node_name self._server_address = server_address self._client_tag = client_tag self._client_address = client_address self._deliverator = deliverator self._send_queue = gevent.queue.Queue() # prime the send queue with messages to be sent as soon # as we connect for connect_message in connect_messages: if not "message-id" in connect_message: connect_message["message-id"] = uuid.uuid1().hex message = message_format(ident=None, control=connect_message, body=None) self._send_queue.put(message) self._req_socket = None self.connected = False
def queue_message_for_send(self, message_control, data=None): """ message control must be a dict If the caller includes a 'message-id' key we will use it, otherwise, we will supply one. return: a gevent queue (zero size Queue) the the reply will be deliverd. """ if not self.connected: raise ResilientClientError( "queue_message_for_send while not connected %s" % ( message_control, ) ) if not "message-id" in message_control: message_control["message-id"] = uuid.uuid1().hex message_id = message_control["message-id"] delivery_channel = self._deliverator.add_request(message_id) message = message_format( ident=None, control=message_control, body=data ) self._send_queue.put(message) return delivery_channel
def _handle_status_disconnected(self): elapsed_time = time.time() - self._status_time if elapsed_time < _handshake_retry_interval: return assert self._dealer_socket is None self._dealer_socket = self._context.socket(zmq.XREQ) self._dealer_socket.setsockopt(zmq.LINGER, 1000) self._log.debug("connecting to server") self._dealer_socket.connect(self._server_address) self._pollster.register_read( self._dealer_socket, self._pollster_callback ) message = { "message-type" : "resilient-server-handshake", "message-id" : uuid.uuid1().hex, "client-tag" : self._client_tag, "client-address" : self._client_address, } message = message_format(ident=None, control=message, body=None) self._pending_message = message self._pending_message_start_time = time.time() self._status = _status_handshaking self._status_time = time.time() self._send_message(message)
def queue_message_for_send(self, message_control, data=None): """ message_control a dictionary, to be sent as JSON data (optional) binary data (possibly a sequence of strings) to be sent in the message queue a message for send (unless we can send it immediately) if message_control does not contain a message-id, we will supply one. """ if not "message-id" in message_control: message_control["message-id"] = uuid.uuid1().hex message = message_format( ident=None, control=message_control, body=data ) if self._status is _status_connected and self._pending_message is None: self._pending_message = message self._pending_message_start_time = time.time() self._send_message(message) else: self._send_queue.append(message)
def queue_message_for_send(self, message_control, data=None): message_ident = message_control.pop("router-ident") self._send_queue.append( message_format( ident=message_ident, control=message_control, body=data ) )
def _complete_handoff(self, message, data, completion_channel): # hand off the message, if "handoff-node-name" in message: assert message["handoff-node-name"] is None, message message["handoff-node-name"] = self._dest_node_name data_writer_greenlets = [ # queue a copy of the message, so each gets a different message-id gevent.spawn( self._hand_off_to_one_data_writer, client, message.copy(), data ) \ for client in self._backup_clients ] # get all replies from the actual clients gevent.joinall(data_writer_greenlets, timeout=_data_writer_timeout) all_ready = all([g.ready() for g in data_writer_greenlets]) if not all_ready: self._log.error("Not all data writer greenlets finished") assert all_ready data_writer_replies = [g.value for g in data_writer_greenlets] # if any data writer has failed, we have failed for data_writer_reply in data_writer_replies: if data_writer_reply["result"] != "success": reply = { "message-type": "handoff-failure", "result": "handoff-failure", "error-message": data_writer_reply["error-message"], } message = message_format(ident=None, control=reply, body=None) completion_channel.put(( message.control, message.body, )) return # if we made it here all the handoffs succeeded, # notify the sender that we are ok so far # by sending one of the data_writer_replies reply = data_writer_replies[0] message = message_format(ident=None, control=reply, body=None) completion_channel.put(( message.control, message.body, ))
def queue_message_for_broadcast(self, message_control, data=None): """ queue a message for send, but do not create a delivery channel: we are not expecting a reply """ if not "message-id" in message_control: message_control["message-id"] = uuid.uuid1().hex message = message_format(ident=None, control=message_control, body=data) self._send_queue.put(message)
def _complete_handoff(self, message, data, completion_channel): # hand off the message, if "handoff-node-name" in message: assert message["handoff-node-name"] is None, message message["handoff-node-name"] = self._dest_node_name data_writer_greenlets = [ # queue a copy of the message, so each gets a different message-id gevent.spawn( self._hand_off_to_one_data_writer, client, message.copy(), data ) \ for client in self._backup_clients ] # get all replies from the actual clients gevent.joinall(data_writer_greenlets, timeout=_data_writer_timeout) all_ready = all([g.ready() for g in data_writer_greenlets]) if not all_ready: self._log.error("Not all data writer greenlets finished") assert all_ready data_writer_replies = [g.value for g in data_writer_greenlets] # if any data writer has failed, we have failed for data_writer_reply in data_writer_replies: if data_writer_reply["result"] != "success": reply = { "message-type" : "handoff-failure", "result" : "handoff-failure", "error-message" : data_writer_reply["error-message"], } message = message_format(ident=None, control=reply, body=None) completion_channel.put((message.control, message.body, )) return # if we made it here all the handoffs succeeded, # notify the sender that we are ok so far # by sending one of the data_writer_replies reply = data_writer_replies[0] message = message_format(ident=None, control=reply, body=None) completion_channel.put((message.control, message.body, ))
def test_single_entry(self): """test adding and removing a single entry""" queue = PriorityQueue() message = message_format( ident=None, control={"priority" : 0}, body=None ) self.assertEqual(len(queue), 0) queue.append((message.control, message.body, )) self.assertEqual(len(queue), 1) retrieved_message, _retrieved_body = queue.popleft() self.assertEqual(retrieved_message, message.control) self.assertEqual(len(queue), 0) self.assertRaises(IndexError, queue.popleft)
def queue_message_for_broadcast(self, message_control, data=None): """ queue a message for send, but do not create a delivery channel: we are not expecting a reply """ if not "message-id" in message_control: message_control["message-id"] = uuid.uuid1().hex message = message_format( ident=None, control=message_control, body=data ) self._send_queue.put(message)
def test_message_order(self): """test that we get messages in the order we expect""" queue = PriorityQueue() messages = [ message_format(ident=None, control={ "priority": 0, "expected-order": 0 }, body=None), message_format(ident=None, control={ "priority": 5, "expected-order": 2 }, body=None), message_format(ident=None, control={ "priority": 3, "expected-order": 1 }, body=None), ] self.assertEqual(len(queue), 0) for message in messages: queue.append(( message.control, message.body, )) self.assertEqual(len(queue), len(messages)) for order in range(len(messages)): retrieved_message, _retrieved_data = queue.popleft() self.assertEqual(order, retrieved_message["expected-order"]) self.assertEqual(len(queue), 0) self.assertRaises(IndexError, queue.popleft)
def _receive_message(self): control = self._rep_socket.recv_json() body = [] while self._rep_socket.rcvmore: body.append(self._rep_socket.recv()) # 2011-04-06 dougfort -- if someone is expecting a list and we only get # one segment, they are going to have to deal with it. if len(body) == 0: body = None elif len(body) == 1: body = body[0] return message_format(ident=None, control=control, body=body)
def queue_message_for_send(self, message_control, data=None): """ message control must be a dict If the caller includes a 'message-id' key we will use it, otherwise, we will supply one. return: a gevent queue (zero size Queue) the the reply will be deliverd. """ if not "message-id" in message_control: message_control["message-id"] = uuid.uuid1().hex self._delivery_queues[message_control["message-id"]] = \ Queue(maxsize=None) self._send_queue.put( message_format(ident=None, control=message_control, body=data)) return self._delivery_queues[message_control["message-id"]]
def test_single_entry(self): """test adding and removing a single entry""" queue = PriorityQueue() message = message_format(ident=None, control={"priority": 0}, body=None) self.assertEqual(len(queue), 0) queue.append(( message.control, message.body, )) self.assertEqual(len(queue), 1) retrieved_message, _retrieved_body = queue.popleft() self.assertEqual(retrieved_message, message.control) self.assertEqual(len(queue), 0) self.assertRaises(IndexError, queue.popleft)
def queue_message_for_send(self, message_control, data=None): """ message control must be a dict If the caller includes a 'message-id' key we will use it, otherwise, we will supply one. return: a gevent queue (zero size Queue) the the reply will be deliverd. """ if not "message-id" in message_control: message_control["message-id"] = uuid.uuid1().hex self._delivery_queues[message_control["message-id"]] = \ Queue(maxsize=None) self._send_queue.put( message_format(ident=None, control=message_control, body=data) ) return self._delivery_queues[message_control["message-id"]]
def _run(self): while True: control = self._pull_socket.recv_json() body = [] while self._pull_socket.rcvmore: body.append(self._pull_socket.recv()) # 2011-04-06 dougfort -- if someone is expecting a list and we # only get one segment, they are going to have to deal with it. if len(body) == 0: body = None elif len(body) == 1: body = body[0] message = message_format(ident=None, control=control, body=body) self._log.debug("received: %s" % (message.control, )) self._deliverator.deliver_reply(message)
def _deliver_failure_reply(self, message_to_send): """ deliver a failure reply to everyone waiting for this socket """ work_message = message_to_send while work_message is not None: reply = { "message-type" : "ack-timeout-reply", "message-id" : work_message.control["message-id"], "result" : "ack timeout", "error-message" : "timeout waiting ack: treating as disconnect" } message = message_format(ident=None, control=reply, body=None) self._deliverator.deliver_reply(message) try: work_message = self._send_queue.get_nowait() except gevent.queue.Empty: work_message = None
def _receive_message(self): try: control = self._pull_socket.recv_json(zmq.NOBLOCK) except zmq.ZMQError: instance = sys.exc_info()[1] if instance.errno == zmq.EAGAIN: self._log.warn("socket would have blocked") return None raise body = [] while self._pull_socket.rcvmore: body.append(self._pull_socket.recv()) # 2011-04-06 dougfort -- if someone is expecting a list and we only get # one segment, they are going to have to deal with it. if len(body) == 0: body = None elif len(body) == 1: body = body[0] return message_format(ident=None, control=control, body=body)
def __init__( self, context, server_node_name, server_address, client_tag, client_address, deliverator, connect_messages=list() ): Greenlet.__init__(self) self._log = logging.getLogger("ResilientClient-%s" % ( server_address, )) self._context = context self._server_node_name = server_node_name self._server_address = server_address self._client_tag = client_tag self._client_address = client_address self._deliverator = deliverator self._send_queue = gevent.queue.Queue() # prime the send queue with messages to be sent as soon # as we connect for connect_message in connect_messages: if not "message-id" in connect_message: connect_message["message-id"] = uuid.uuid1().hex message = message_format( ident=None, control=connect_message, body=None ) self._send_queue.put(message) self._req_socket = None self.connected = False
def queue_message_for_send(self, message_control, data=None): self._send_queue.append( message_format(ident=None, control=message_control, body=data))
class DealerClient(object): """ a class that manages a zeromq DEALER (aka XREQ) socket as a client """ def __init__(self, context, address, receive_queue): self._log = logging.getLogger("DealerClient-%s" % (address, )) self._dealer_socket = context.socket(zmq.XREQ) self._dealer_socket.setsockopt(zmq.LINGER, 1000) self._log.debug("connecting") self._dealer_socket.connect(address) self._send_queue = deque() self._receive_queue = receive_queue def register(self, pollster): pollster.register_read_or_write(self._dealer_socket, self._pollster_callback) def unregister(self, pollster): pollster.unregister(self._dealer_socket) def close(self): self._dealer_socket.close() def queue_message_for_send(self, message_control, data=None): self._send_queue.append( message_format(ident=None, control=message_control, body=data)) def _pollster_callback(self, _active_socket, readable, writable): # push our output first while writable: try: message = self._send_queue.popleft() except IndexError: break self._send_message(message) # if we have input, read it and queue it if readable: message = self._receive_message() # if we get None, that means the socket would have blocked # go back and wait for more if message is None: return None self._receive_queue.append(( message.control, message.body, )) def _send_message(self, message): self._log.debug("sending message: %s" % (message.control, )) # don't send a zero size body if type(message.body) not in [ list, tuple, type(None), ]: if len(message.body) == 0: message = message._replace(body=None) else: message = message._replace(body=[ message.body, ]) if message.body is None: self._dealer_socket.send_json(message.control) else: self._dealer_socket.send_json(message.control, zmq.SNDMORE) for segment in message.body[:-1]: self._dealer_socket.send(segment, zmq.SNDMORE) self._dealer_socket.send(message.body[-1]) def _receive_message(self): try: control = self._dealer_socket.recv_json(zmq.NOBLOCK) except zmq.ZMQError, instance: if instance.errno == zmq.EAGAIN: self._log.warn("socket would have blocked") return None raise body = [] while self._dealer_socket.rcvmore: body.append(self._dealer_socket.recv()) # 2011-04-06 dougfort -- if someone is expecting a list and we only get # one segment, they are going to have to deal with it. if len(body) == 0: body = None elif len(body) == 1: body = body[0] return message_format(ident=None, control=control, body=body)
def send_reply(self, message_control, data=None): message = message_format(ident=None, control=message_control, body=data) self._send_message(message)
class RouterServer(object): """ a class that manages a zeromq ROUTER (aka XREP) socket as a server """ def __init__(self, context, address, receive_queue): self._log = logging.getLogger("RouterServer-%s" % (address, )) # we need a valid path for IPC sockets if address.startswith("ipc://"): prepare_ipc_path(address) self._router_socket = context.socket(zmq.XREP) self._router_socket.setsockopt(zmq.LINGER, 1000) self._log.debug("binding") self._router_socket.bind(address) self._send_queue = deque() self._receive_queue = receive_queue def register(self, pollster): pollster.register_read_or_write(self._router_socket, self._pollster_callback) def unregister(self, pollster): pollster.unregister(self._router_socket) def close(self): self._router_socket.close() def queue_message_for_send(self, message_control, data=None): message_ident = message_control.pop("router-ident") self._send_queue.append( message_format(ident=message_ident, control=message_control, body=data)) def _pollster_callback(self, _active_socket, readable, writable): # push our output first, no point in taking on new work # when we haven't sent our old stuff while writable: try: message = self._send_queue.popleft() except IndexError: break self._send_message(message) # if we have input, read it and handle it if readable: message = self._receive_message() # if we get None, that means the socket would have blocked # go back and wait for more if message is None: return # stuff the ident into the message, we'll want it back # if the caller tries to send something message.control["router-ident"] = message.ident self._receive_queue.append(( message.control, message.body, )) def _send_message(self, message): self._log.debug("sending message: %s" % (message.control, )) self._router_socket.send(message.ident, zmq.SNDMORE) # don't send a zero size body if type(message.body) not in [ list, tuple, type(None), ]: if len(message.body) == 0: message = message._replace(body=None) else: message = message._replace(body=[ message.body, ]) if message.body is None: self._router_socket.send_json(message.control) else: self._router_socket.send_json(message.control, zmq.SNDMORE) for segment in message.body[:-1]: self._router_socket.send(segment, zmq.SNDMORE) self._router_socket.send(message.body[-1]) def _receive_message(self): try: ident = self._router_socket.recv(zmq.NOBLOCK) except zmq.ZMQError, instance: if instance.errno == zmq.EAGAIN: self._log.warn("socket would have blocked") return None raise assert self._router_socket.rcvmore, \ "Unexpected missing message control part." control = self._router_socket.recv_json() body = [] while self._router_socket.rcvmore: body.append(self._router_socket.recv()) # 2011-04-06 dougfort -- if someone is expecting a list and we only get # one segment, they are going to have to deal with it. if len(body) == 0: body = None elif len(body) == 1: body = body[0] return message_format(ident=ident, control=control, body=body)
def queue_message_for_send(self, message_control, data=None): message_ident = message_control.pop("router-ident") self._send_queue.append( message_format(ident=message_ident, control=message_control, body=data))
def queue_message_for_send(self, message_control, data=None): self._send_queue.append( message_format(ident=None, control=message_control, body=data) )