def get_message(self, correlation_id): try: while correlation_id not in self.replies: self.consumer.connection.drain_events(timeout=self.timeout) body, message = self.replies.pop(correlation_id) self.provider.handle_message(body, message) except socket.timeout: # TODO: this conflates an rpc timeout with a socket read timeout. # a better rpc proxy implementation would recover from a socket # timeout if the rpc timeout had not yet been reached timeout_error = RpcTimeout(self.timeout) event = self.provider._reply_events.pop(correlation_id) event.send_exception(timeout_error) # timeout is implemented using socket timeout, so when it # fires the connection is closed and must be re-established self._setup_consumer() except (IOError, ConnectionError) as exc: # in case this was a temporary error, attempt to reconnect # and try again. if we fail to reconnect, the error will bubble self._setup_consumer() self.get_message(correlation_id) except KeyboardInterrupt as exc: event = self.provider._reply_events.pop(correlation_id) event.send_exception(exc) # exception may have killed the connection self._setup_consumer()
def get_message(self, correlation_id): try: while correlation_id not in self.replies: self.consumer.channel.connection.client.drain_events( timeout=self.timeout) body, message = self.replies.pop(correlation_id) self.provider.handle_message(body, message) except socket.timeout: timeout_error = RpcTimeout(self.timeout) event = self.provider._reply_events.pop(correlation_id) event.send_exception(timeout_error) # timeout is implemented using socket timeout, so when it # fires the connection is closed, causing the reply queue # to be deleted self._setup_consumer() except (IOError, ConnectionError) as exc: for event in self.provider._reply_events.values(): rpc_connection_error = RpcConnectionError( 'Disconnected while waiting for reply: %s', exc) event.send_exception(rpc_connection_error) self.provider._reply_events.clear() # In case this was a temporary error, attempt to reconnect. If # we fail, the connection error will bubble. self._setup_consumer() except KeyboardInterrupt as exc: event = self.provider._reply_events.pop(correlation_id) event.send_exception(exc) # exception may have killed the connection self._setup_consumer()
def wait(self, timeout=None): timeout = timeout if timeout is not None else self.timeout try: res = self.get(timeout=timeout) if res['exc']: raise res['exc'] return res['result'] except Empty: raise RpcTimeout(timeout)
def queue_iterator(queue, no_ack=False, timeout=None): channel = queue.channel consumer = Consumer(channel, queues=[queue], no_ack=no_ack) try: for body, msg in drain_consumer(consumer, limit=None, timeout=timeout, ignore_timeouts=False): yield body, msg except socket.timeout: if timeout is not None: # we raise a different exception type here because we bubble out # to our caller, but `socket.timeout` errors get caught if # our connection is "ensured" with `kombu.Connection.ensure`; # the reference to the connection is destroyed so it can't be # closed later - see https://github.com/celery/kombu/blob/v3.0.4/ # kombu/connection.py#L446 raise RpcTimeout(timeout) raise
def get_message(self, correlation_id): start_time = time.time() stop_waiting = False # retrieve agreed heartbeat interval of both party by query the server HEARTBEAT_INTERVAL = self.consumer.connection.get_heartbeat_interval() RATE = lambda: min( 2 + abs(time.time() - start_time) * .75 / HEARTBEAT_INTERVAL, # noqa: E731, E501 HEARTBEAT_INTERVAL / 3) true_timeout = lambda: abs(start_time + self.timeout - time.time( ) # noqa: E731 ) if self.timeout is not None else None remaining_timeout = lambda: ( min( abs(start_time + self.timeout - time.time() ), # noqa: E731, E501 HEARTBEAT_INTERVAL / RATE()) if self.timeout is not None else HEARTBEAT_INTERVAL / RATE()) if self.heartbeat else true_timeout() is_timed_out = lambda: abs( time.time() - start_time # noqa: E731 ) > self.timeout if self.timeout is not None else False timed_out_err_msg = "Timeout after: {}".format(self.timeout) while correlation_id not in self.replies: recover_connection = False try: if self.heartbeat: try: self.consumer.connection.heartbeat_check() except (ConnectionError, socket.error, IOError) as exc: _logger.info( "Heart beat failed." "System will auto recover broken connection, %s: %s", type(exc).__name__, exc.args[0]) raise else: _logger.debug("Heart beat OK") self.consumer.connection.drain_events( timeout=remaining_timeout()) except socket.timeout: # if socket timeout happen here, send a heartbeat and keep looping # until self.timeout is reached or correlation_id is found pass except (ConnectionError, socket.error, IOError) as exc: # in case this was a temporary error, attempt to reconnect # and try again. if we fail to reconnect, the error will bubble # wait till connection stable before retry if isinstance(exc, IOError) and not isinstance(exc, socket.error): # check only certain IOError will attempt to reconnect else reraise if exc.args[ 0] not in EXPECTED_IOERR_MSG_SET: # check error.message raise if not is_timed_out(): try: # try to recover connection if there is still time recover_connection = True _logger.debug( "Stabilizing connection to message broker due to error," " {}: {}".format(type(exc).__name__, exc.args[0])) self._setup_connection() self.connection.ensure_connection( max_retries=2, timeout=true_timeout()) if self.connection.connected is True: self._setup_consumer() recover_connection = False # continue the loop to start send a heartbeat # and wait for result with this new connection else: err_msg = "Unable to stabilizing connection after error," \ " {}: {}".format(type(exc).__name__, exc.args[0]) _logger.debug(err_msg) event = self.provider._reply_events.pop( correlation_id) event.send_exception(ConnectionError(err_msg)) stop_waiting = True except socket.timeout: timed_out_err_msg = \ "Timeout after stabilizing connection: {}".format( self.timeout) # continue the loop to try to recover connection until # either self.timeout is reached or correlation_id is found except (ConnectionError, socket.error) as exc2: err_msg = "Error during stabilizing connection, {}: {}".format( type(exc2).__name__, exc2.args[0]) _logger.debug(err_msg) event = self.provider._reply_events.pop(correlation_id) event.send_exception(ConnectionError(err_msg)) recover_connection = True stop_waiting = True except KeyboardInterrupt as exc: event = self.provider._reply_events.pop(correlation_id) event.send_exception(exc) recover_connection = True stop_waiting = True finally: if correlation_id in self.replies: body, message = self.replies.pop(correlation_id) self.provider.handle_message(body, message) stop_waiting = True else: if is_timed_out() is True: _logger.debug(timed_out_err_msg) timeout_error = RpcTimeout(timed_out_err_msg) event = self.provider._reply_events.pop(correlation_id) event.send_exception(timeout_error) stop_waiting = True # always try to recover the connection before exit if this flag is True if recover_connection: try: if self.connection.connected is False: self._setup_connection() self._setup_consumer() except socket.error as exc: _logger.debug( "Socket error during setup consumer, %s: %s", type(exc).__name__, exc.args[0]) if stop_waiting: # stop waiting for result, break the loop break else: # other thread may have receive the message corresponding to # this correlation_id before enter wait loop body, message = self.replies.pop(correlation_id) self.provider.handle_message(body, message)