def _on_error(self, description):
"""Invoked by the eventloop if the send operation fails for reasons
other than timeout and nack.
"""
msg = ("{name} message sent to {target} failed:"
" {reason}".format(name=self.name,
target=self.target,
reason=description))
LOG.warning("%s", msg)
self._error = exceptions.MessageDeliveryFailure(msg)
self._cleanup()
self._wakeup.set()
def reconnect(self, retry=None):
"""Handles reconnecting and re-establishing sessions and queues.
Will retry up to retry number of times.
retry = None or -1 means to retry forever
retry = 0 means no retry
retry = N means N retries
"""
delay = 1
attempt = 0
loop_forever = False
if retry is None or retry < 0:
loop_forever = True
while True:
self._disconnect()
attempt += 1
broker = six.next(self.brokers)
try:
self._connect(broker)
except qpid_exceptions.MessagingError as e:
msg_dict = dict(e=e,
delay=delay,
retry=retry,
broker=broker)
if not loop_forever and attempt > retry:
msg = _('Unable to connect to AMQP server on '
'%(broker)s after %(retry)d '
'tries: %(e)s') % msg_dict
LOG.error(msg)
raise exceptions.MessageDeliveryFailure(msg)
else:
msg = _("Unable to connect to AMQP server on %(broker)s: "
"%(e)s. Sleeping %(delay)s seconds") % msg_dict
LOG.error(msg)
time.sleep(delay)
delay = min(delay + 1, 5)
else:
LOG.info(_('Connected to AMQP server on %s'), broker['host'])
break
self.session = self.connection.session()
if self.consumers:
consumers = self.consumers
self.consumers = {}
for consumer in six.itervalues(consumers):
consumer.reconnect(self.session)
self._register_consumer(consumer)
LOG.debug("Re-established AMQP queues")
def _on_timeout(self):
"""Invoked by the eventloop when the send fails to complete before the
timeout is reached.
"""
self.timer = None
msg = ("{name} message sent to {target} failed: timed"
" out".format(name=self.name, target=self.target))
LOG.warning("%s", msg)
# Only raise a MessagingTimeout if the caller has explicitly specified
# a timeout.
self._error = exceptions.MessagingTimeout(msg) \
if self.message.ttl else \
exceptions.MessageDeliveryFailure(msg)
self._cleanup()
self._wakeup.set()
def _on_ack(self, state, info):
"""Called by eventloop thread when the ack/nack is received from the
peer.
"""
if state != pyngus.SenderLink.ACCEPTED:
# TODO(kgiusti): could retry if deadline not hit
msg = ("{name} message send to {target} failed: remote"
" disposition: {disp}, info:"
"{info}".format(name=self.name,
target=self.target,
disp=state,
info=info))
self._error = exceptions.MessageDeliveryFailure(msg)
LOG.warning("%s", msg)
self._cleanup()
self._wakeup.set()
def _on_error(self, description):
"""Invoked by the eventloop if the send operation fails for reasons
other than timeout and nack.
"""
if self._wakeup.is_set():
LOG.debug("Message send error occurred after send completed: %s",
str(description))
return
msg = ("{name} message sent to {target} failed:"
" {reason}".format(name=self.name,
target=self.target,
reason=description))
self._error = exceptions.MessageDeliveryFailure(msg)
self._cleanup()
self._wakeup.set()
def _callback(link, handle, state, info):
if state == pyngus.SenderLink.ACCEPTED: # message received
if not reply_expected:
# can wake up the sender now
result = {"status": "OK"}
result_queue.put(result)
else:
# we will wake up the sender when the reply message is
# received. See Replies.message_received()
pass
else: # send failed/rejected/etc
msg = "Message send failed: remote disposition: %s, info: %s"
exc = exceptions.MessageDeliveryFailure(msg % (state, info))
result = {"status": "ERROR", "error": exc}
if reply_expected:
# no response will be received, so cancel the correlation
self._replies.cancel_response(msg_id)
result_queue.put(result)
def _on_timeout(self):
"""Invoked by the eventloop when the send fails to complete before the
timeout is reached.
"""
if self._wakeup.is_set():
LOG.debug("Message send timeout occurred after send completed")
return
self.timer = None
if self.message.ttl:
msg = ("{name} message sent to {target} failed: timed"
" out".format(name=self.name, target=self.target))
self._error = exceptions.MessagingTimeout(msg)
else:
msg = ("{name} message sent to {target} failed:"
" undeliverable".format(name=self.name, target=self.target))
self._error = exceptions.MessageDeliveryFailure(msg)
self._cleanup()
self._wakeup.set()
def ensure(self, method, retry=None,
recoverable_error_callback=None, error_callback=None,
timeout_is_error=True):
"""Will retry up to retry number of times.
retry = None means use the value of rabbit_max_retries
retry = -1 means to retry forever
retry = 0 means no retry
retry = N means N retries
NOTE(sileht): Must be called within the connection lock
"""
current_pid = os.getpid()
if self._initial_pid != current_pid:
LOG.warn(_LW("Process forked after connection established! "
"This can result in unpredictable behavior. "
"See: http://docs.openstack.org/developer/"
"oslo_messaging/transport.html"))
self._initial_pid = current_pid
if retry is None:
retry = self.max_retries
if retry is None or retry < 0:
retry = None
def on_error(exc, interval):
LOG.debug("Received recoverable error from kombu:",
exc_info=True)
recoverable_error_callback and recoverable_error_callback(exc)
interval = (self.kombu_reconnect_delay + interval
if self.kombu_reconnect_delay > 0
else interval)
info = {'err_str': exc, 'sleep_time': interval}
info.update(self.connection.info())
if 'Socket closed' in six.text_type(exc):
LOG.error(_LE('AMQP server %(hostname)s:%(port)s closed'
' the connection. Check login credentials:'
' %(err_str)s'), info)
else:
LOG.error(_LE('AMQP server on %(hostname)s:%(port)s is '
'unreachable: %(err_str)s. Trying again in '
'%(sleep_time)d seconds.'), info)
# XXX(nic): when reconnecting to a RabbitMQ cluster
# with mirrored queues in use, the attempt to release the
# connection can hang "indefinitely" somewhere deep down
# in Kombu. Blocking the thread for a bit prior to
# release seems to kludge around the problem where it is
# otherwise reproduceable.
# TODO(sileht): Check if this is useful since we
# use kombu for HA connection, the interval_step
# should sufficient, because the underlying kombu transport
# connection object freed.
if self.kombu_reconnect_delay > 0:
LOG.trace('Delaying reconnect for %1.1f seconds ...',
self.kombu_reconnect_delay)
time.sleep(self.kombu_reconnect_delay)
def on_reconnection(new_channel):
"""Callback invoked when the kombu reconnects and creates
a new channel, we use it the reconfigure our consumers.
"""
self._set_current_channel(new_channel)
for consumer in self._consumers:
consumer.declare(self)
LOG.info(_LI('Reconnected to AMQP server on '
'%(hostname)s:%(port)s via [%(transport)s] client'),
self.connection.info())
def execute_method(channel):
self._set_current_channel(channel)
method()
# NOTE(sileht): Some dummy driver like the in-memory one doesn't
# have notion of recoverable connection, so we must raise the original
# exception like kombu does in this case.
has_modern_errors = hasattr(
self.connection.transport, 'recoverable_connection_errors',
)
if has_modern_errors:
recoverable_errors = (
self.connection.recoverable_channel_errors +
self.connection.recoverable_connection_errors)
else:
recoverable_errors = ()
try:
autoretry_method = self.connection.autoretry(
execute_method, channel=self.channel,
max_retries=retry,
errback=on_error,
interval_start=self.interval_start or 1,
interval_step=self.interval_stepping,
on_revive=on_reconnection,
)
ret, channel = autoretry_method()
self._set_current_channel(channel)
return ret
except recoverable_errors as exc:
LOG.debug("Received recoverable error from kombu:",
exc_info=True)
error_callback and error_callback(exc)
self._set_current_channel(None)
# NOTE(sileht): number of retry exceeded and the connection
# is still broken
info = {'err_str': exc, 'retry': retry}
info.update(self.connection.info())
msg = _('Unable to connect to AMQP server on '
'%(hostname)s:%(port)s after %(retry)s '
'tries: %(err_str)s') % info
LOG.error(msg)
raise exceptions.MessageDeliveryFailure(msg)
except rpc_amqp.AMQPDestinationNotFound:
# NOTE(sileht): we must reraise this without
# trigger error_callback
raise
except Exception as exc:
error_callback and error_callback(exc)
raise
