def receive_message(self, socket): try: reply_id = socket.recv() empty = socket.recv() assert empty == b'', 'Bad format: empty delimiter expected' msg_type = socket.recv_string() assert msg_type is not None, 'Bad format: msg type expected' msg_id = None if msg_type != zmq_names.CALL_TYPE: msg_id = socket.recv_string() context = socket.recv_pyobj() message = socket.recv_pyobj() LOG.info(_LI("Received %(msg_type)s message %(msg)s") % {"msg_type": msg_type, "msg": str(message)}) if msg_type == zmq_names.CALL_TYPE: return zmq_incoming_message.ZmqIncomingRequest( self.server, context, message, socket, reply_id, self.poller) elif msg_type in (zmq_names.CAST_TYPES + zmq_names.NOTIFY_TYPES): return RouterIncomingMessage( self.server, context, message, socket, reply_id, msg_id, self.poller) else: LOG.error(_LE("Unknown message type: %s") % msg_type) except zmq.ZMQError as e: LOG.error(_LE("Receiving message failed: %s") % str(e))
def _process_incoming(self, incoming): message = incoming[0] try: message.acknowledge() except Exception: LOG.exception(_LE("Can not acknowledge message. Skip processing")) return failure = None try: res = self.dispatcher.dispatch(message) except rpc_dispatcher.ExpectedException as e: failure = e.exc_info LOG.debug(u'Expected exception during message handling (%s)', e) except Exception: # current sys.exc_info() content can be overriden # by another exception raised by a log handler during # LOG.exception(). So keep a copy and delete it later. failure = sys.exc_info() LOG.exception(_LE('Exception during message handling')) try: if failure is None: message.reply(res) else: message.reply(failure=failure) except Exception: LOG.exception(_LE("Can not send reply for message")) finally: # NOTE(dhellmann): Remove circular object reference # between the current stack frame and the traceback in # exc_info. del failure
def _process_incoming(self, incoming): message = incoming[0] try: message.acknowledge() #回答mes except Exception: LOG.exception(_LE("Can not acknowledge message. Skip processing")) return failure = None try: res = self.dispatcher.dispatch(message) except rpc_dispatcher.ExpectedException as e: failure = e.exc_info LOG.debug(u'Expected exception during message handling (%s)', e) except Exception: # current sys.exc_info() content can be overridden # by another exception raised by a log handler during # LOG.exception(). So keep a copy and delete it later. failure = sys.exc_info() LOG.exception(_LE('Exception during message handling')) try: if failure is None: message.reply(res) else: message.reply(failure=failure) except Exception: LOG.exception(_LE("Can not send reply for message")) finally: # NOTE(dhellmann): Remove circular object reference # between the current stack frame and the traceback in # exc_info. del failure
def on_error(exc, interval): LOG.debug(_("Received recoverable error from kombu:"), exc_info=True) recoverable_error_callback and recoverable_error_callback(exc) interval = (self.kombu_reconnect_delay + interval if self.kombu_reconnect_delay > 0 else interval) info = {'err_str': exc, 'sleep_time': interval} info.update(self.connection.info()) if 'Socket closed' in six.text_type(exc): LOG.error(_LE('AMQP server %(hostname)s:%(port)s closed' ' the connection. Check login credentials:' ' %(err_str)s'), info) else: LOG.error(_LE('AMQP server on %(hostname)s:%(port)s is ' 'unreachable: %(err_str)s. Trying again in ' '%(sleep_time)d seconds.'), info) # XXX(nic): when reconnecting to a RabbitMQ cluster # with mirrored queues in use, the attempt to release the # connection can hang "indefinitely" somewhere deep down # in Kombu. Blocking the thread for a bit prior to # release seems to kludge around the problem where it is # otherwise reproduceable. # TODO(sileht): Check if this is useful since we # use kombu for HA connection, the interval_step # should sufficient, because the underlying kombu transport # connection object freed. if self.kombu_reconnect_delay > 0: time.sleep(self.kombu_reconnect_delay)
def receive_message(self, socket): try: empty = socket.recv() assert empty == b'', 'Bad format: empty delimiter expected' reply_id = socket.recv() message_type = int(socket.recv()) message_id = socket.recv() context = socket.recv_loaded() message = socket.recv_loaded() LOG.debug("[%(host)s] Received %(msg_type)s message %(msg_id)s", {"host": self.host, "msg_type": zmq_names.message_type_str(message_type), "msg_id": message_id}) if message_type == zmq_names.CALL_TYPE: return zmq_incoming_message.ZmqIncomingMessage( context, message, reply_id, message_id, socket, self.sender ) elif message_type in zmq_names.NON_BLOCKING_TYPES: return zmq_incoming_message.ZmqIncomingMessage(context, message) else: LOG.error(_LE("Unknown message type: %s"), zmq_names.message_type_str(message_type)) except (zmq.ZMQError, AssertionError) as e: LOG.error(_LE("Receiving message failure: %s"), str(e))
def consume_in_thread(self): """Runs the ZmqProxy service.""" ipc_dir = CONF.rpc_zmq_ipc_dir consume_in = "tcp://%s:%s" % \ (CONF.rpc_zmq_bind_address, CONF.rpc_zmq_port) consumption_proxy = InternalContext(None) try: os.makedirs(ipc_dir) except os.error: if not os.path.isdir(ipc_dir): with excutils.save_and_reraise_exception(): LOG.error(_LE("Required IPC directory does not exist at" " %s"), ipc_dir) try: self.register(consumption_proxy, consume_in, zmq.PULL) except zmq.ZMQError: if os.access(ipc_dir, os.X_OK): with excutils.save_and_reraise_exception(): LOG.error(_LE("Permission denied to IPC directory at" " %s"), ipc_dir) with excutils.save_and_reraise_exception(): LOG.error(_LE("Could not create ZeroMQ receiver daemon. " "Socket may already be in use.")) super(ZmqProxy, self).consume_in_thread()
def receive_message(self, socket): try: reply_id = socket.recv() empty = socket.recv() assert empty == b'', 'Bad format: empty delimiter expected' request = socket.recv_pyobj() LOG.info( _LI("Received %(msg_type)s message %(msg)s") % { "msg_type": request.msg_type, "msg": str(request.message) }) if request.msg_type == zmq_names.CALL_TYPE: return zmq_incoming_message.ZmqIncomingRequest( self.server, request.context, request.message, socket, reply_id, self.poller) elif request.msg_type in zmq_names.NON_BLOCKING_TYPES: return RouterIncomingMessage(self.server, request.context, request.message, socket, reply_id, request.message_id, self.poller) else: LOG.error(_LE("Unknown message type: %s") % request.msg_type) except zmq.ZMQError as e: LOG.error(_LE("Receiving message failed: %s") % str(e))
def receive_message(self, socket): try: reply_id, msg_type, message_id, context, message = \ self._receive_request(socket) LOG.debug( "[%(host)s] Received %(msg_type)s message %(msg_id)s", { "host": self.host, "msg_type": zmq_names.message_type_str(msg_type), "msg_id": message_id }) if msg_type == zmq_names.CALL_TYPE or \ msg_type in zmq_names.NON_BLOCKING_TYPES: ack_sender = self.ack_sender \ if self.conf.oslo_messaging_zmq.rpc_use_acks else None reply_sender = self.reply_sender \ if msg_type == zmq_names.CALL_TYPE else None return zmq_incoming_message.ZmqIncomingMessage( context, message, reply_id, message_id, socket, ack_sender, reply_sender) else: LOG.error(_LE("Unknown message type: %s"), zmq_names.message_type_str(msg_type)) except (zmq.ZMQError, AssertionError, ValueError) as e: LOG.error(_LE("Receiving message failed: %s"), str(e))
def receive_message(self, socket): try: empty = socket.recv() assert empty == b'', 'Bad format: empty delimiter expected' reply_id = socket.recv() message_type = int(socket.recv()) message_id = socket.recv() context = socket.recv_loaded() message = socket.recv_loaded() LOG.debug( "[%(host)s] Received %(msg_type)s message %(msg_id)s", { "host": self.host, "msg_type": zmq_names.message_type_str(message_type), "msg_id": message_id }) if message_type == zmq_names.CALL_TYPE: return zmq_incoming_message.ZmqIncomingMessage( context, message, reply_id, message_id, socket, self.sender) elif message_type in zmq_names.NON_BLOCKING_TYPES: return zmq_incoming_message.ZmqIncomingMessage( context, message) else: LOG.error(_LE("Unknown message type: %s"), zmq_names.message_type_str(message_type)) except (zmq.ZMQError, AssertionError, ValueError) as e: LOG.error(_LE("Receiving message failure: %s"), str(e))
def on_error(exc, interval): LOG.debug(_("Received recoverable error from kombu:"), exc_info=True) recoverable_error_callback and recoverable_error_callback(exc) interval = (self.driver_conf.kombu_reconnect_delay + interval if self.driver_conf.kombu_reconnect_delay > 0 else interval) info = {'err_str': exc, 'sleep_time': interval} info.update(self.connection.info()) if 'Socket closed' in six.text_type(exc): LOG.error(_LE('AMQP server %(hostname)s:%(port)d closed' ' the connection. Check login credentials:' ' %(err_str)s'), info) else: LOG.error(_LE('AMQP server on %(hostname)s:%(port)d is ' 'unreachable: %(err_str)s. Trying again in ' '%(sleep_time)d seconds.'), info) # XXX(nic): when reconnecting to a RabbitMQ cluster # with mirrored queues in use, the attempt to release the # connection can hang "indefinitely" somewhere deep down # in Kombu. Blocking the thread for a bit prior to # release seems to kludge around the problem where it is # otherwise reproduceable. # TODO(sileht): Check if this is useful since we # use kombu for HA connection, the interval_step # should sufficient, because the underlying kombu transport # connection object freed. if self.driver_conf.kombu_reconnect_delay > 0: time.sleep(self.driver_conf.kombu_reconnect_delay)
def consume_in_thread(self): """Runs the ZmqProxy service.""" ipc_dir = CONF.rpc_zmq_ipc_dir consume_in = "tcp://%s:%s" % \ (CONF.rpc_zmq_bind_address, CONF.rpc_zmq_port) consumption_proxy = InternalContext(None) try: os.makedirs(ipc_dir) except os.error: if not os.path.isdir(ipc_dir): with excutils.save_and_reraise_exception(): LOG.error( _LE("Required IPC directory does not exist at" " %s"), ipc_dir) try: self.register(consumption_proxy, consume_in, zmq.PULL) except zmq.ZMQError: if os.access(ipc_dir, os.X_OK): with excutils.save_and_reraise_exception(): LOG.error( _LE("Permission denied to IPC directory at" " %s"), ipc_dir) with excutils.save_and_reraise_exception(): LOG.error( _LE("Could not create ZeroMQ receiver daemon. " "Socket may already be in use.")) super(ZmqProxy, self).consume_in_thread()
def __init__(self, *args, **kwargs): message = kwargs.get("message") if message['method'] is None: errmsg = _LE("No method specified for RPC call") LOG.error(_LE("No method specified for RPC call")) raise KeyError(errmsg) super(RpcRequest, self).__init__(*args, **kwargs)
def import_zmq(zmq_concurrency="eventlet"): _raise_error_if_invalid_config_value(zmq_concurrency) imported_zmq = importutils.try_import(ZMQ_MODULES[zmq_concurrency], default="zmq") if imported_zmq is None: errmsg = _LE("ZeroMQ not found!") LOG.error(_LE("ZeroMQ not found!")) raise ImportError(errmsg) return imported_zmq
def __init__(self, *args, **kwargs): message = kwargs.get("message") if message["method"] is None: errmsg = _LE("No method specified for RPC call") LOG.error(_LE("No method specified for RPC call")) raise KeyError(errmsg) self.timeout = kwargs.pop("timeout") assert self.timeout is not None, "Timeout should be specified!" super(RpcRequest, self).__init__(*args, **kwargs)
def __init__(self, *args, **kwargs): message = kwargs.get("message") if message['method'] is None: errmsg = _LE("No method specified for RPC call") LOG.error(_LE("No method specified for RPC call")) raise KeyError(errmsg) self.timeout = kwargs.pop("timeout") assert self.timeout is not None, "Timeout should be specified!" super(RpcRequest, self).__init__(*args, **kwargs)
def receive_message(self, socket): try: reply_id, msg_type, message_id, context, message = \ self._receive_request(socket) if msg_type == zmq_names.CALL_TYPE or \ msg_type in zmq_names.NON_BLOCKING_TYPES: ack_sender = self.ack_sender \ if self.conf.oslo_messaging_zmq.rpc_use_acks else None reply_sender = self.reply_sender \ if msg_type == zmq_names.CALL_TYPE else None message = zmq_incoming_message.ZmqIncomingMessage( context, message, reply_id, message_id, socket, ack_sender, reply_sender, self.messages_cache) # drop a duplicate message if message_id in self.messages_cache: LOG.warning( _LW("[%(host)s] Dropping duplicate %(msg_type)s " "message %(msg_id)s"), { "host": self.host, "msg_type": zmq_names.message_type_str(msg_type), "msg_id": message_id }) # NOTE(gdavoian): send yet another ack for the non-CALL # message, since the old one might be lost; # for the CALL message also try to resend its reply # (of course, if it was already obtained and cached). message._acknowledge() if msg_type == zmq_names.CALL_TYPE: message._reply_from_cache() return None self.messages_cache.add(message_id) LOG.debug( "[%(host)s] Received %(msg_type)s message %(msg_id)s", { "host": self.host, "msg_type": zmq_names.message_type_str(msg_type), "msg_id": message_id }) # NOTE(gdavoian): send an immediate ack, since it may # be too late to wait until the message will be # dispatched and processed by a RPC server message._acknowledge() return message else: LOG.error(_LE("Unknown message type: %s"), zmq_names.message_type_str(msg_type)) except (zmq.ZMQError, AssertionError, ValueError) as e: LOG.error(_LE("Receiving message failure: %s"), str(e))
def import_zmq(zmq_concurrency='eventlet'): _raise_error_if_invalid_config_value(zmq_concurrency) imported_zmq = importutils.try_import(ZMQ_MODULES[zmq_concurrency], default='zmq') if imported_zmq is None: errmsg = _LE("ZeroMQ not found!") LOG.error(_LE("ZeroMQ not found!")) raise ImportError(errmsg) return imported_zmq
def connect(self, host): """Connect to host and start the AMQP protocol.""" addr = socket.getaddrinfo(host.hostname, host.port, socket.AF_INET, socket.SOCK_STREAM) if not addr: key = "%s:%i" % (host.hostname, host.port) error = "Invalid peer address '%s'" % key LOG.error(_LE("Invalid peer address '%s'"), key) self._handler.socket_error(error) return my_socket = socket.socket(addr[0][0], addr[0][1], addr[0][2]) my_socket.setblocking(0) # 0=non-blocking my_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) try: my_socket.connect(addr[0][4]) except socket.error as e: if e.errno != errno.EINPROGRESS: error = "Socket connect failure '%s'" % str(e) LOG.error(_LE("Socket connect failure '%s'"), str(e)) self._handler.socket_error(error) return self.socket = my_socket props = self._properties.copy() if pyngus.VERSION >= (2, 0, 0): # configure client authentication # props['x-server'] = False if host.username: props['x-username'] = host.username props['x-password'] = host.password or "" c = self._container.create_connection(self.name, self._handler, props) c.user_context = self self.connection = c if pyngus.VERSION < (2, 0, 0): # older versions of pyngus requires manual SASL configuration: # determine the proper SASL mechanism: PLAIN if a username/password # is present, else ANONYMOUS pn_sasl = self.connection.pn_sasl if host.username: password = host.password if host.password else "" pn_sasl.plain(host.username, password) else: pn_sasl.mechanisms("ANONYMOUS") # TODO(kgiusti): server if accepting inbound connections pn_sasl.client() self.connection.open()
def connect_to_address(self, address): stype = zmq_names.socket_type_str(self.socket_type) try: LOG.info(_LI("Connecting %(stype)s id %(id)s to %(address)s"), {"stype": stype, "id": self.handle.identity, "address": address}) self.connect(address) except zmq.ZMQError as e: errmsg = _LE("Failed connecting %(stype) to %(address)s: %(e)s")\ % (stype, address, e) LOG.error(_LE("Failed connecting %(stype) to %(address)s: %(e)s"), (stype, address, e)) raise rpc_common.RPCException(errmsg)
def _process_incoming(self, incoming): try: not_processed_messages = self.dispatcher.dispatch(incoming) except Exception: not_processed_messages = set(incoming) LOG.error(_LE('Exception during message handling'), exc_info=True) for m in incoming: try: if m in not_processed_messages and self._allow_requeue: m.requeue() else: m.acknowledge() except Exception: LOG.error(_LE("Fail to ack/requeue message"), exc_info=True)
def _process_incoming(self, incoming): try: not_processed_messages = self.dispatcher.dispatch(incoming) except Exception: not_processed_messages = set(incoming) LOG.exception(_LE('Exception during messages handling.')) for m in incoming: try: if m in not_processed_messages and self._allow_requeue: m.requeue() else: m.acknowledge() except Exception: LOG.exception(_LE("Fail to ack/requeue message."))
def _process_incoming(self, incoming): res = notify_dispatcher.NotificationResult.REQUEUE try: res = self.dispatcher.dispatch(incoming) except Exception: LOG.error(_LE('Exception during message handling'), exc_info=True) try: if (res == notify_dispatcher.NotificationResult.REQUEUE and self._allow_requeue): incoming.requeue() else: incoming.acknowledge() except Exception: LOG.error(_LE("Fail to ack/requeue message"), exc_info=True)
def __init__(self, *args, **kwargs): message = kwargs.get("message") if message['method'] is None: errmsg = _LE("No method specified for RPC call") LOG.error(_LE("No method specified for RPC call")) raise KeyError(errmsg) self.timeout = kwargs.pop("timeout") assert self.timeout is not None, "Timeout should be specified!" if not isinstance(self.timeout, int) and self.timeout is not None: raise ValueError("timeout must be an integer, not {0}".format( type(self.timeout))) super(RpcRequest, self).__init__(*args, **kwargs)
def _process_incoming(self, incoming): message = incoming[0] try: res = self.dispatcher.dispatch(message) except Exception: LOG.error(_LE('Exception during message handling'), exc_info=True) try: if (res == notify_dispatcher.NotificationResult.REQUEUE and self._allow_requeue): message.requeue() else: message.acknowledge() except Exception: LOG.error(_LE("Fail to ack/requeue message"), exc_info=True)
def subscribe_socket(self, socket_type): try: socket = zmq_socket.ZmqRandomPortSocket(self.conf, self.context, socket_type) self.sockets.append(socket) LOG.debug( "Run %(stype)s consumer on %(addr)s:%(port)d", {"stype": zmq_names.socket_type_str(socket_type), "addr": socket.bind_address, "port": socket.port}, ) self.host = zmq_address.combine_address(self.conf.rpc_zmq_host, socket.port) self.poller.register(socket, self.receive_message) return socket except zmq.ZMQError as e: errmsg = _LE("Failed binding to port %(port)d: %(e)s") % (self.port, e) LOG.error(_LE("Failed binding to port %(port)d: %(e)s"), (self.port, e)) raise rpc_common.RPCException(errmsg)
def wrapped(*args, **kwargs): try: return fn(*args, **kwargs) except Exception as e: LOG.exception( _LE('An exception occurred processing ' 'the API call: %s ') % e)
def consume(self, sock): # TODO(ewindisch): use zero-copy (i.e. references, not copying) data = sock.recv() LOG.debug("CONSUMER RECEIVED DATA: %s", data) proxy = self.proxies[sock] if data[2] == b'cast': # Legacy protocol packenv = data[3] ctx, msg = _deserialize(packenv) request = rpc_common.deserialize_msg(msg) ctx = RpcContext.unmarshal(ctx) elif data[2] == b'impl_zmq_v2': packenv = data[4:] msg = unflatten_envelope(packenv) request = rpc_common.deserialize_msg(msg) # Unmarshal only after verifying the message. ctx = RpcContext.unmarshal(data[3]) else: LOG.error(_LE("ZMQ Envelope version unsupported or unknown.")) return self.pool.spawn_n(self.process, proxy, ctx, request)
def _done(self): """If the connection came from a pool, clean it up and put it back. If it did not come from a pool, close it. """ if self.connection: if self.pooled: # Reset the connection so it's ready for the next caller # to grab from the pool try: self.connection.reset() except Exception: LOG.exception(_LE("Fail to reset the connection, drop it")) try: self.connection.close() except Exception: pass self.connection = self.connection_pool.create() finally: self.connection_pool.put(self.connection) else: try: self.connection.close() except Exception: pass self.connection = None
def __init__(self, conf, *args, **kwargs): if redis is None: raise ImportError(_LE("Redis package is not available!")) super(MatchmakerRedisBase, self).__init__(conf, *args, **kwargs) self.conf.register_opts(matchmaker_redis_opts, "matchmaker_redis")
def consume(self, timeout=None): """Receive up to 'max_fetch_messages' messages. :param timeout: poll timeout in seconds """ def _raise_timeout(exc): raise driver_common.Timeout(str(exc)) timer = driver_common.DecayingTimer(duration=timeout) timer.start() poll_timeout = (self.consumer_timeout if timeout is None else min(timeout, self.consumer_timeout)) while True: if self._consume_loop_stopped: return try: return self._poll_messages(poll_timeout) except kafka.errors.ConsumerTimeout as exc: poll_timeout = timer.check_return( _raise_timeout, exc, maximum=self.consumer_timeout) except Exception: LOG.exception(_LE("Failed to consume messages")) return
def _process_incoming(self, incoming): message = incoming[0] message.acknowledge() try: res = self.dispatcher.dispatch(message) except rpc_dispatcher.ExpectedException as e: LOG.debug(u'Expected exception during message handling (%s)', e.exc_info[1]) message.reply(failure=e.exc_info) except Exception as e: # current sys.exc_info() content can be overriden # by another exception raise by a log handler during # LOG.exception(). So keep a copy and delete it later. exc_info = sys.exc_info() try: LOG.exception(_LE('Exception during message handling: %s'), e) message.reply(failure=exc_info) finally: # NOTE(dhellmann): Remove circular object reference # between the current stack frame and the traceback in # exc_info. del exc_info else: try: message.reply(res) except Exception: LOG.Exception("Can not send reply for message %s", message)
def connect_to_address(self, address): if address in self.connections: return stype = zmq_names.socket_type_str(self.socket_type) try: LOG.info(_LI("Connecting %(stype)s id %(id)s to %(address)s"), {"stype": stype, "id": self.handle.identity, "address": address}) self.connect(address) except zmq.ZMQError as e: errmsg = _LE("Failed connecting %(stype)s to %(address)s: %(e)s") \ % {"stype": stype, "address": address, "e": e} LOG.error(_LE("Failed connecting %(stype)s to %(address)s: %(e)s"), {"stype": stype, "address": address, "e": e}) raise rpc_common.RPCException(errmsg)
def __init__(self, *args, **kwargs): message = kwargs.get("message") if message['method'] is None: errmsg = _LE("No method specified for RPC call") LOG.error(_LE("No method specified for RPC call")) raise KeyError(errmsg) self.timeout = kwargs.pop("timeout") assert self.timeout is not None, "Timeout should be specified!" if not isinstance(self.timeout, int) and self.timeout is not None: raise ValueError( "timeout must be an integer, not {0}" .format(type(self.timeout))) super(RpcRequest, self).__init__(*args, **kwargs)
def _done(self): """If the connection came from a pool, clean it up and put it back. If it did not come from a pool, close it. """ if self.connection: if self.pooled: # Reset the connection so it's ready for the next caller # to grab from the pool try: self.connection.reset() except Exception: LOG.exception(_LE("Fail to reset the connection, drop it")) try: self.connection.close() except Exception as exc: LOG.debug("pooled conn close failure (ignored): %s", str(exc)) self.connection = self.connection_pool.create() finally: self.connection_pool.put(self.connection) else: try: self.connection.close() except Exception as exc: LOG.debug("pooled conn close failure (ignored): %s", str(exc)) self.connection = None
def _process_incoming(self, incoming): message = incoming[0] try: res = self.dispatcher.dispatch(message) except Exception: LOG.exception(_LE('Exception during message handling.')) res = notify_dispatcher.NotificationResult.REQUEUE try: if (res == notify_dispatcher.NotificationResult.REQUEUE and self._allow_requeue): message.requeue() else: message.acknowledge() except Exception: LOG.exception(_LE("Fail to ack/requeue message."))
def receive_message(self, socket): try: msg_type = socket.recv_string() assert msg_type is not None, 'Bad format: msg type expected' context = socket.recv_pyobj() message = socket.recv_pyobj() LOG.debug("Received %(msg_type)s message %(msg)s", {"msg_type": msg_type, "msg": str(message)}) if msg_type in (zmq_names.CAST_TYPES + zmq_names.NOTIFY_TYPES): return PullIncomingMessage(self.server, context, message) else: LOG.error(_LE("Unknown message type: %s"), msg_type) except zmq.ZMQError as e: LOG.error(_LE("Receiving message failed: %s"), str(e))
def consume(self, timeout=None): """Receive up to 'max_fetch_messages' messages. :param timeout: poll timeout in seconds """ def _raise_timeout(exc): raise driver_common.Timeout(exc.message) timer = driver_common.DecayingTimer(duration=timeout) timer.start() poll_timeout = (self.consumer_timeout if timeout is None else min( timeout, self.consumer_timeout)) while True: if self._consume_loop_stopped: return try: return self._poll_messages(poll_timeout) except kafka.errors.ConsumerTimeout as exc: poll_timeout = timer.check_return( _raise_timeout, exc, maximum=self.consumer_timeout) except Exception: LOG.exception(_LE("Failed to consume messages")) return
def poll(self): while not self._thread_exit_event.is_set(): try: self.conn.consume() except Exception: LOG.exception(_LE("Failed to process incoming message, " "retrying..."))
def receive_message(self, socket): try: request = self._receive_request(socket) if not request: return None LOG.debug("Received %(type)s, %(id)s, %(target)s", {"type": request.msg_type, "id": request.message_id, "target": request.target}) if request.msg_type not in zmq_names.MULTISEND_TYPES: LOG.error(_LE("Unknown message type: %s"), request.msg_type) else: return SubIncomingMessage(request, socket, self.poller) except zmq.ZMQError as e: LOG.error(_LE("Receiving message failed: %s"), str(e))
def do_notify(ext): try: ext.obj.notify(ctxt, msg, priority, retry or self.retry) except Exception as e: _LOG.exception(_LE("Problem '%(e)s' attempting to send to " "notification system. Payload=%(payload)s"), dict(e=e, payload=payload))
def _connect_to_host(self, socket, host, target): address = zmq_address.get_tcp_direct_address(host) LOG.info(address) stype = zmq_names.socket_type_str(self.socket_type) try: LOG.info(_LI("Connecting %(stype)s to %(address)s for %(target)s") % {"stype": stype, "address": address, "target": target}) socket.connect(address) except zmq.ZMQError as e: errmsg = _LE("Failed connecting %(stype) to %(address)s: %(e)s")\ % (stype, address, e) LOG.error(_LE("Failed connecting %(stype) to %(address)s: %(e)s") % (stype, address, e)) raise rpc_common.RPCException(errmsg)
def __init__(self, conf, context, socket_type, host=None, high_watermark=0, identity=None): super(ZmqRandomPortSocket, self).__init__(conf, context, socket_type, immediate=False, high_watermark=high_watermark, identity=identity) self.bind_address = zmq_address.get_tcp_random_address(self.conf) if host is None: host = conf.oslo_messaging_zmq.rpc_zmq_host try: self.port = self.handle.bind_to_random_port( self.bind_address, min_port=conf.oslo_messaging_zmq.rpc_zmq_min_port, max_port=conf.oslo_messaging_zmq.rpc_zmq_max_port, max_tries=conf.oslo_messaging_zmq.rpc_zmq_bind_port_retries) self.connect_address = zmq_address.combine_address(host, self.port) except zmq.ZMQBindError: LOG.error(_LE("Random ports range exceeded!")) raise ZmqPortBusy(port_number=0)
def __init__(self, conf, context, socket_type, host, port, high_watermark=0, identity=None): super(ZmqFixedPortSocket, self).__init__(conf, context, socket_type, immediate=False, high_watermark=high_watermark, identity=identity) self.connect_address = zmq_address.combine_address(host, port) self.bind_address = zmq_address.get_tcp_direct_address( zmq_address.combine_address( conf.oslo_messaging_zmq.rpc_zmq_bind_address, port)) self.host = host self.port = port try: self.handle.bind(self.bind_address) except zmq.ZMQError as e: LOG.exception(e) LOG.error(_LE("Chosen port %d is being busy.") % self.port) raise ZmqPortBusy(port_number=port)
def connect_to_address(self, address): if address in self.connections: return stype = zmq_names.socket_type_str(self.socket_type) sid = self.handle.identity try: LOG.debug("Connecting %(stype)s socket %(sid)s to %(address)s", { "stype": stype, "sid": sid, "address": address }) self.connect(address) except zmq.ZMQError as e: LOG.error( _LE("Failed connecting %(stype)s-%(sid)s to " "%(address)s: %(e)s"), { "stype": stype, "sid": sid, "address": address, "e": e }) raise rpc_common.RPCException( "Failed connecting %(stype)s-%(sid)s to %(address)s: %(e)s" % { "stype": stype, "sid": sid, "address": address, "e": e })
def _done(self): """If the connection came from a pool, clean it up and put it back. If it did not come from a pool, close it. """ if self.connection: if self.pooled: #reset connection,reset 失败则create and put # Reset the connection so it's ready for the next caller # to grab from the pool try: self.connection.reset() except Exception: LOG.exception(_LE("Fail to reset the connection, drop it")) try: self.connection.close() except Exception: pass self.connection = self.connection_pool.create() finally: self.connection_pool.put(self.connection) else: try: self.connection.close() except Exception: pass self.connection = None
def _get_response(self, ctx, proxy, topic, data): """Process a curried message and cast the result to topic.""" LOG.debug("Running func with context: %s", ctx.to_dict()) data.setdefault('version', None) data.setdefault('args', {}) try: if not data.get("method"): raise KeyError result = proxy.dispatch(ctx, data) return ConsumerBase.normalize_reply(result, ctx.replies) except greenlet.GreenletExit: # ignore these since they are just from shutdowns pass except rpc_common.ClientException as e: LOG.debug("Expected exception during message handling (%s)", e._exc_info[1]) return { 'exc': rpc_common.serialize_remote_exception(e._exc_info, log_failure=False) } except Exception: LOG.error(_LE("Exception during message handling")) return { 'exc': rpc_common.serialize_remote_exception(sys.exc_info()) }
def receiver_remote_closed(self, receiver, pn_condition): """This is a Pyngus callback, invoked by Pyngus when the peer of this receiver link has initiated closing the connection. """ # TODO(kgiusti) Unclear if this error will ever occur (as opposed to # the Connection failing instead). Log for now, possibly implement a # recovery strategy if necessary. LOG.error(_LE("Reply subscription closed by peer: %s"), (pn_condition or "no error given"))
def __init__(self, pattern_name): """Construct exception object :param pattern_name: Message type name from zmq_names :type pattern_name: str """ errmsg = _LE("Sending pattern %s is unsupported.") % pattern_name super(UnsupportedSendPattern, self).__init__(errmsg)
def receive_message(self, socket): try: context, message = self._receive_request(socket) if not message: return None return zmq_incoming_message.ZmqIncomingMessage(context, message) except (zmq.ZMQError, AssertionError) as e: LOG.error(_LE("Receiving message failed: %s"), str(e))
def connect_to_address(self, address): stype = zmq_names.socket_type_str(self.socket_type) try: LOG.info(_LI("Connecting %(stype)s to %(address)s"), {"stype": stype, "address": address}) if six.PY3: self.setsockopt_string(zmq.IDENTITY, str(uuid.uuid1())) else: self.handle.identity = str(uuid.uuid1()) self.connect(address) except zmq.ZMQError as e: errmsg = _LE("Failed connecting %(stype) to %(address)s: %(e)s")\ % (stype, address, e) LOG.error(_LE("Failed connecting %(stype) to %(address)s: %(e)s"), (stype, address, e)) raise rpc_common.RPCException(errmsg)
def consume(self): """Fetch the message and pass it to the callback object.""" message = self.receiver.fetch() try: self._unpack_json_msg(message) self.callback(QpidMessage(self.session, message)) except Exception: LOG.exception(_LE("Failed to process message... skipping it.")) self.session.acknowledge(message)
def _post_dispatch(self, incoming, requeues): for m in incoming: try: if requeues and m in requeues: m.requeue() else: m.acknowledge() except Exception: LOG.error(_LE("Fail to ack/requeue message"), exc_info=True)
def _create_ipc_dirs(self): ipc_dir = self.conf.rpc_zmq_ipc_dir try: os.makedirs("%s/fanout" % ipc_dir) except os.error: if not os.path.isdir(ipc_dir): with excutils.save_and_reraise_exception(): LOG.error(_LE("Required IPC directory does not exist at" " %s"), ipc_dir)