def get_message(self, correlation_id):
try:
while correlation_id not in self.replies:
self.consumer.connection.drain_events(timeout=self.timeout)
body, message = self.replies.pop(correlation_id)
self.provider.handle_message(body, message)
except socket.timeout:
# TODO: this conflates an rpc timeout with a socket read timeout.
# a better rpc proxy implementation would recover from a socket
# timeout if the rpc timeout had not yet been reached
timeout_error = RpcTimeout(self.timeout)
event = self.provider._reply_events.pop(correlation_id)
event.send_exception(timeout_error)
# timeout is implemented using socket timeout, so when it
# fires the connection is closed and must be re-established
self._setup_consumer()
except (IOError, ConnectionError) as exc:
# in case this was a temporary error, attempt to reconnect
# and try again. if we fail to reconnect, the error will bubble
self._setup_consumer()
self.get_message(correlation_id)
except KeyboardInterrupt as exc:
event = self.provider._reply_events.pop(correlation_id)
event.send_exception(exc)
# exception may have killed the connection
self._setup_consumer()
def get_message(self, correlation_id):
try:
while correlation_id not in self.replies:
self.consumer.channel.connection.client.drain_events(
timeout=self.timeout)
body, message = self.replies.pop(correlation_id)
self.provider.handle_message(body, message)
except socket.timeout:
timeout_error = RpcTimeout(self.timeout)
event = self.provider._reply_events.pop(correlation_id)
event.send_exception(timeout_error)
# timeout is implemented using socket timeout, so when it
# fires the connection is closed, causing the reply queue
# to be deleted
self._setup_consumer()
except (IOError, ConnectionError) as exc:
for event in self.provider._reply_events.values():
rpc_connection_error = RpcConnectionError(
'Disconnected while waiting for reply: %s', exc)
event.send_exception(rpc_connection_error)
self.provider._reply_events.clear()
# In case this was a temporary error, attempt to reconnect. If
# we fail, the connection error will bubble.
self._setup_consumer()
except KeyboardInterrupt as exc:
event = self.provider._reply_events.pop(correlation_id)
event.send_exception(exc)
# exception may have killed the connection
self._setup_consumer()
def wait(self, timeout=None):
timeout = timeout if timeout is not None else self.timeout
try:
res = self.get(timeout=timeout)
if res['exc']:
raise res['exc']
return res['result']
except Empty:
raise RpcTimeout(timeout)
def queue_iterator(queue, no_ack=False, timeout=None):
channel = queue.channel
consumer = Consumer(channel, queues=[queue], no_ack=no_ack)
try:
for body, msg in drain_consumer(consumer,
limit=None,
timeout=timeout,
ignore_timeouts=False):
yield body, msg
except socket.timeout:
if timeout is not None:
# we raise a different exception type here because we bubble out
# to our caller, but `socket.timeout` errors get caught if
# our connection is "ensured" with `kombu.Connection.ensure`;
# the reference to the connection is destroyed so it can't be
# closed later - see https://github.com/celery/kombu/blob/v3.0.4/
# kombu/connection.py#L446
raise RpcTimeout(timeout)
raise
def get_message(self, correlation_id):
start_time = time.time()
stop_waiting = False
# retrieve agreed heartbeat interval of both party by query the server
HEARTBEAT_INTERVAL = self.consumer.connection.get_heartbeat_interval()
RATE = lambda: min(
2 + abs(time.time() - start_time) * .75 /
HEARTBEAT_INTERVAL, # noqa: E731, E501
HEARTBEAT_INTERVAL / 3)
true_timeout = lambda: abs(start_time + self.timeout - time.time(
) # noqa: E731
) if self.timeout is not None else None
remaining_timeout = lambda: (
min(
abs(start_time + self.timeout - time.time()
), # noqa: E731, E501
HEARTBEAT_INTERVAL / RATE()) if self.timeout is not None else
HEARTBEAT_INTERVAL / RATE()) if self.heartbeat else true_timeout()
is_timed_out = lambda: abs(
time.time() - start_time # noqa: E731
) > self.timeout if self.timeout is not None else False
timed_out_err_msg = "Timeout after: {}".format(self.timeout)
while correlation_id not in self.replies:
recover_connection = False
try:
if self.heartbeat:
try:
self.consumer.connection.heartbeat_check()
except (ConnectionError, socket.error, IOError) as exc:
_logger.info(
"Heart beat failed."
"System will auto recover broken connection, %s: %s",
type(exc).__name__, exc.args[0])
raise
else:
_logger.debug("Heart beat OK")
self.consumer.connection.drain_events(
timeout=remaining_timeout())
except socket.timeout:
# if socket timeout happen here, send a heartbeat and keep looping
# until self.timeout is reached or correlation_id is found
pass
except (ConnectionError, socket.error, IOError) as exc:
# in case this was a temporary error, attempt to reconnect
# and try again. if we fail to reconnect, the error will bubble
# wait till connection stable before retry
if isinstance(exc,
IOError) and not isinstance(exc, socket.error):
# check only certain IOError will attempt to reconnect else reraise
if exc.args[
0] not in EXPECTED_IOERR_MSG_SET: # check error.message
raise
if not is_timed_out():
try: # try to recover connection if there is still time
recover_connection = True
_logger.debug(
"Stabilizing connection to message broker due to error,"
" {}: {}".format(type(exc).__name__, exc.args[0]))
self._setup_connection()
self.connection.ensure_connection(
max_retries=2, timeout=true_timeout())
if self.connection.connected is True:
self._setup_consumer()
recover_connection = False
# continue the loop to start send a heartbeat
# and wait for result with this new connection
else:
err_msg = "Unable to stabilizing connection after error," \
" {}: {}".format(type(exc).__name__, exc.args[0])
_logger.debug(err_msg)
event = self.provider._reply_events.pop(
correlation_id)
event.send_exception(ConnectionError(err_msg))
stop_waiting = True
except socket.timeout:
timed_out_err_msg = \
"Timeout after stabilizing connection: {}".format(
self.timeout)
# continue the loop to try to recover connection until
# either self.timeout is reached or correlation_id is found
except (ConnectionError, socket.error) as exc2:
err_msg = "Error during stabilizing connection, {}: {}".format(
type(exc2).__name__, exc2.args[0])
_logger.debug(err_msg)
event = self.provider._reply_events.pop(correlation_id)
event.send_exception(ConnectionError(err_msg))
recover_connection = True
stop_waiting = True
except KeyboardInterrupt as exc:
event = self.provider._reply_events.pop(correlation_id)
event.send_exception(exc)
recover_connection = True
stop_waiting = True
finally:
if correlation_id in self.replies:
body, message = self.replies.pop(correlation_id)
self.provider.handle_message(body, message)
stop_waiting = True
else:
if is_timed_out() is True:
_logger.debug(timed_out_err_msg)
timeout_error = RpcTimeout(timed_out_err_msg)
event = self.provider._reply_events.pop(correlation_id)
event.send_exception(timeout_error)
stop_waiting = True
# always try to recover the connection before exit if this flag is True
if recover_connection:
try:
if self.connection.connected is False:
self._setup_connection()
self._setup_consumer()
except socket.error as exc:
_logger.debug(
"Socket error during setup consumer, %s: %s",
type(exc).__name__, exc.args[0])
if stop_waiting: # stop waiting for result, break the loop
break
else:
# other thread may have receive the message corresponding to
# this correlation_id before enter wait loop
body, message = self.replies.pop(correlation_id)
self.provider.handle_message(body, message)
