def _on_error(self, description): """Invoked by the eventloop if the send operation fails for reasons other than timeout and nack. """ msg = ("{name} message sent to {target} failed:" " {reason}".format(name=self.name, target=self.target, reason=description)) LOG.warning("%s", msg) self._error = exceptions.MessageDeliveryFailure(msg) self._cleanup() self._wakeup.set()
def reconnect(self, retry=None): """Handles reconnecting and re-establishing sessions and queues. Will retry up to retry number of times. retry = None or -1 means to retry forever retry = 0 means no retry retry = N means N retries """ delay = 1 attempt = 0 loop_forever = False if retry is None or retry < 0: loop_forever = True while True: self._disconnect() attempt += 1 broker = six.next(self.brokers) try: self._connect(broker) except qpid_exceptions.MessagingError as e: msg_dict = dict(e=e, delay=delay, retry=retry, broker=broker) if not loop_forever and attempt > retry: msg = _('Unable to connect to AMQP server on ' '%(broker)s after %(retry)d ' 'tries: %(e)s') % msg_dict LOG.error(msg) raise exceptions.MessageDeliveryFailure(msg) else: msg = _("Unable to connect to AMQP server on %(broker)s: " "%(e)s. Sleeping %(delay)s seconds") % msg_dict LOG.error(msg) time.sleep(delay) delay = min(delay + 1, 5) else: LOG.info(_('Connected to AMQP server on %s'), broker['host']) break self.session = self.connection.session() if self.consumers: consumers = self.consumers self.consumers = {} for consumer in six.itervalues(consumers): consumer.reconnect(self.session) self._register_consumer(consumer) LOG.debug("Re-established AMQP queues")
def _on_timeout(self): """Invoked by the eventloop when the send fails to complete before the timeout is reached. """ self.timer = None msg = ("{name} message sent to {target} failed: timed" " out".format(name=self.name, target=self.target)) LOG.warning("%s", msg) # Only raise a MessagingTimeout if the caller has explicitly specified # a timeout. self._error = exceptions.MessagingTimeout(msg) \ if self.message.ttl else \ exceptions.MessageDeliveryFailure(msg) self._cleanup() self._wakeup.set()
def _on_ack(self, state, info): """Called by eventloop thread when the ack/nack is received from the peer. """ if state != pyngus.SenderLink.ACCEPTED: # TODO(kgiusti): could retry if deadline not hit msg = ("{name} message send to {target} failed: remote" " disposition: {disp}, info:" "{info}".format(name=self.name, target=self.target, disp=state, info=info)) self._error = exceptions.MessageDeliveryFailure(msg) LOG.warning("%s", msg) self._cleanup() self._wakeup.set()
def _on_error(self, description): """Invoked by the eventloop if the send operation fails for reasons other than timeout and nack. """ if self._wakeup.is_set(): LOG.debug("Message send error occurred after send completed: %s", str(description)) return msg = ("{name} message sent to {target} failed:" " {reason}".format(name=self.name, target=self.target, reason=description)) self._error = exceptions.MessageDeliveryFailure(msg) self._cleanup() self._wakeup.set()
def _callback(link, handle, state, info): if state == pyngus.SenderLink.ACCEPTED: # message received if not reply_expected: # can wake up the sender now result = {"status": "OK"} result_queue.put(result) else: # we will wake up the sender when the reply message is # received. See Replies.message_received() pass else: # send failed/rejected/etc msg = "Message send failed: remote disposition: %s, info: %s" exc = exceptions.MessageDeliveryFailure(msg % (state, info)) result = {"status": "ERROR", "error": exc} if reply_expected: # no response will be received, so cancel the correlation self._replies.cancel_response(msg_id) result_queue.put(result)
def _on_timeout(self): """Invoked by the eventloop when the send fails to complete before the timeout is reached. """ if self._wakeup.is_set(): LOG.debug("Message send timeout occurred after send completed") return self.timer = None if self.message.ttl: msg = ("{name} message sent to {target} failed: timed" " out".format(name=self.name, target=self.target)) self._error = exceptions.MessagingTimeout(msg) else: msg = ("{name} message sent to {target} failed:" " undeliverable".format(name=self.name, target=self.target)) self._error = exceptions.MessageDeliveryFailure(msg) self._cleanup() self._wakeup.set()
def ensure(self, method, retry=None, recoverable_error_callback=None, error_callback=None, timeout_is_error=True): """Will retry up to retry number of times. retry = None means use the value of rabbit_max_retries retry = -1 means to retry forever retry = 0 means no retry retry = N means N retries NOTE(sileht): Must be called within the connection lock """ current_pid = os.getpid() if self._initial_pid != current_pid: LOG.warn(_LW("Process forked after connection established! " "This can result in unpredictable behavior. " "See: http://docs.openstack.org/developer/" "oslo_messaging/transport.html")) self._initial_pid = current_pid if retry is None: retry = self.max_retries if retry is None or retry < 0: retry = None def on_error(exc, interval): LOG.debug("Received recoverable error from kombu:", exc_info=True) recoverable_error_callback and recoverable_error_callback(exc) interval = (self.kombu_reconnect_delay + interval if self.kombu_reconnect_delay > 0 else interval) info = {'err_str': exc, 'sleep_time': interval} info.update(self.connection.info()) if 'Socket closed' in six.text_type(exc): LOG.error(_LE('AMQP server %(hostname)s:%(port)s closed' ' the connection. Check login credentials:' ' %(err_str)s'), info) else: LOG.error(_LE('AMQP server on %(hostname)s:%(port)s is ' 'unreachable: %(err_str)s. Trying again in ' '%(sleep_time)d seconds.'), info) # XXX(nic): when reconnecting to a RabbitMQ cluster # with mirrored queues in use, the attempt to release the # connection can hang "indefinitely" somewhere deep down # in Kombu. Blocking the thread for a bit prior to # release seems to kludge around the problem where it is # otherwise reproduceable. # TODO(sileht): Check if this is useful since we # use kombu for HA connection, the interval_step # should sufficient, because the underlying kombu transport # connection object freed. if self.kombu_reconnect_delay > 0: LOG.trace('Delaying reconnect for %1.1f seconds ...', self.kombu_reconnect_delay) time.sleep(self.kombu_reconnect_delay) def on_reconnection(new_channel): """Callback invoked when the kombu reconnects and creates a new channel, we use it the reconfigure our consumers. """ self._set_current_channel(new_channel) for consumer in self._consumers: consumer.declare(self) LOG.info(_LI('Reconnected to AMQP server on ' '%(hostname)s:%(port)s via [%(transport)s] client'), self.connection.info()) def execute_method(channel): self._set_current_channel(channel) method() # NOTE(sileht): Some dummy driver like the in-memory one doesn't # have notion of recoverable connection, so we must raise the original # exception like kombu does in this case. has_modern_errors = hasattr( self.connection.transport, 'recoverable_connection_errors', ) if has_modern_errors: recoverable_errors = ( self.connection.recoverable_channel_errors + self.connection.recoverable_connection_errors) else: recoverable_errors = () try: autoretry_method = self.connection.autoretry( execute_method, channel=self.channel, max_retries=retry, errback=on_error, interval_start=self.interval_start or 1, interval_step=self.interval_stepping, on_revive=on_reconnection, ) ret, channel = autoretry_method() self._set_current_channel(channel) return ret except recoverable_errors as exc: LOG.debug("Received recoverable error from kombu:", exc_info=True) error_callback and error_callback(exc) self._set_current_channel(None) # NOTE(sileht): number of retry exceeded and the connection # is still broken info = {'err_str': exc, 'retry': retry} info.update(self.connection.info()) msg = _('Unable to connect to AMQP server on ' '%(hostname)s:%(port)s after %(retry)s ' 'tries: %(err_str)s') % info LOG.error(msg) raise exceptions.MessageDeliveryFailure(msg) except rpc_amqp.AMQPDestinationNotFound: # NOTE(sileht): we must reraise this without # trigger error_callback raise except Exception as exc: error_callback and error_callback(exc) raise