def reconnect(self): """Handles reconnecting and re-establishing queues""" if self.connection: try: self.connection.close() except self.connection.connection_errors: pass time.sleep(1) self.connection = kombu.connection.BrokerConnection(**self.params) if self.memory_transport: # Kludge to speed up tests. self.connection.transport.polling_interval = 0.0 self.consumer_num = itertools.count(1) try: self.connection.ensure_connection( errback=self.connect_error, max_retries=self.max_retries, interval_start=self.interval_start, interval_step=self.interval_stepping, interval_max=self.interval_max) except self.connection.connection_errors, e: # We should only get here if max_retries is set. We'll go # ahead and exit in this case. err_str = str(e) max_retries = self.max_retries LOG.error( _('Unable to connect to AMQP server ' 'after %(max_retries)d tries: %(err_str)s') % locals()) sys.exit(1)
def msg_reply(msg_id, reply=None, failure=None, ending=False): """Sends a reply or an error on the channel signified by msg_id. Failure should be a sys.exc_info() tuple. """ if failure: message = str(failure[1]) tb = traceback.format_exception(*failure) LOG.error(_("Returning exception %s to caller"), message) LOG.error(tb) failure = (failure[0].__name__, str(failure[1]), tb) with ConnectionPool.item() as conn: publisher = DirectPublisher(connection=conn, msg_id=msg_id) try: msg = {'result': reply, 'failure': failure} if ending: msg['ending'] = True publisher.send(msg) except TypeError: msg = { 'result': dict( (k, repr(v)) for k, v in reply.__dict__.iteritems()), 'failure': failure } if ending: msg['ending'] = True publisher.send(msg) publisher.close()
def msg_reply(msg_id, connection_pool, reply=None, failure=None, ending=False): """Sends a reply or an error on the channel signified by msg_id. Failure should be a sys.exc_info() tuple. """ with ConnectionContext(connection_pool) as conn: if failure: message = str(failure[1]) tb = traceback.format_exception(*failure) LOG.error(_("Returning exception %s to caller"), message) LOG.error(tb) failure = (failure[0].__name__, str(failure[1]), tb) try: msg = {'result': reply, 'failure': failure} except TypeError: msg = { 'result': dict( (k, repr(v)) for k, v in reply.__dict__.iteritems()), 'failure': failure } if ending: msg['ending'] = True conn.direct_send(msg_id, msg)
def __init__(self, *args, **kwargs): max_retries = FLAGS.rabbit_max_retries sleep_time = FLAGS.rabbit_retry_interval tries = 0 while True: tries += 1 if tries > 1: time.sleep(sleep_time) # backoff for next retry attempt.. if there is one sleep_time += FLAGS.rabbit_retry_backoff if sleep_time > 30: sleep_time = 30 try: super(Consumer, self).__init__(*args, **kwargs) self.failed_connection = False break except Exception as e: # Catching all because carrot sucks self.failed_connection = True if max_retries > 0 and tries == max_retries: break fl_host = FLAGS.rabbit_host fl_port = FLAGS.rabbit_port fl_intv = sleep_time LOG.error( _('AMQP server on %(fl_host)s:%(fl_port)d is' ' unreachable: %(e)s. Trying again in %(fl_intv)d' ' seconds.') % locals()) if self.failed_connection: LOG.error( _('Unable to connect to AMQP server ' 'after %(tries)d tries. Shutting down.') % locals()) sys.exit(1)
def reconnect(self): """Handles reconnecting and re-establishing queues""" if self.connection: try: self.connection.close() except self.connection.connection_errors: pass time.sleep(1) self.connection = kombu.connection.BrokerConnection(**self.params) if self.memory_transport: # Kludge to speed up tests. self.connection.transport.polling_interval = 0.0 self.consumer_num = itertools.count(1) try: self.connection.ensure_connection(errback=self.connect_error, max_retries=self.max_retries, interval_start=self.interval_start, interval_step=self.interval_stepping, interval_max=self.interval_max) except self.connection.connection_errors, e: # We should only get here if max_retries is set. We'll go # ahead and exit in this case. err_str = str(e) max_retries = self.max_retries LOG.error(_('Unable to connect to AMQP server ' 'after %(max_retries)d tries: %(err_str)s') % locals()) sys.exit(1)
def msg_reply(msg_id, reply=None, failure=None, ending=False): """Sends a reply or an error on the channel signified by msg_id. Failure should be a sys.exc_info() tuple. """ if failure: message = str(failure[1]) tb = traceback.format_exception(*failure) LOG.error(_("Returning exception %s to caller"), message) LOG.error(tb) failure = (failure[0].__name__, str(failure[1]), tb) with ConnectionPool.item() as conn: publisher = DirectPublisher(connection=conn, msg_id=msg_id) try: msg = {'result': reply, 'failure': failure} if ending: msg['ending'] = True publisher.send(msg) except TypeError: msg = {'result': dict((k, repr(v)) for k, v in reply.__dict__.iteritems()), 'failure': failure} if ending: msg['ending'] = True publisher.send(msg) publisher.close()
def fetch(self, no_ack=None, auto_ack=None, enable_callbacks=False): """Wraps the parent fetch with some logic for failed connection.""" # TODO(vish): the logic for failed connections and logging should be # refactored into some sort of connection manager object try: if self.failed_connection: # NOTE(vish): connection is defined in the parent class, we can # recreate it as long as we create the backend too # pylint: disable=W0201 self.connection = Connection.recreate() self.backend = self.connection.create_backend() self.declare() return super(Consumer, self).fetch(no_ack, auto_ack, enable_callbacks) if self.failed_connection: LOG.error(_('Reconnected to queue')) self.failed_connection = False # NOTE(vish): This is catching all errors because we really don't # want exceptions to be logged 10 times a second if some # persistent failure occurs. except Exception, e: # pylint: disable=W0703 if not self.failed_connection: LOG.exception(_('Failed to fetch message from queue: %s' % e)) self.failed_connection = True
def __init__(self, *args, **kwargs): max_retries = FLAGS.rabbit_max_retries sleep_time = FLAGS.rabbit_retry_interval tries = 0 while True: tries += 1 if tries > 1: time.sleep(sleep_time) # backoff for next retry attempt.. if there is one sleep_time += FLAGS.rabbit_retry_backoff if sleep_time > 30: sleep_time = 30 try: super(Consumer, self).__init__(*args, **kwargs) self.failed_connection = False break except Exception as e: # Catching all because carrot sucks self.failed_connection = True if max_retries > 0 and tries == max_retries: break fl_host = FLAGS.rabbit_host fl_port = FLAGS.rabbit_port fl_intv = sleep_time LOG.error(_('AMQP server on %(fl_host)s:%(fl_port)d is' ' unreachable: %(e)s. Trying again in %(fl_intv)d' ' seconds.') % locals()) if self.failed_connection: LOG.error(_('Unable to connect to AMQP server ' 'after %(tries)d tries. Shutting down.') % locals()) sys.exit(1)
def connect_error(self, exc, interval): """Callback when there are connection re-tries by kombu""" info = self.params.copy() info['intv'] = interval info['e'] = exc LOG.error(_('AMQP server on %(hostname)s:%(port)d is' ' unreachable: %(e)s. Trying again in %(intv)d' ' seconds.') % info)
def reconnect(self): """Handles reconnecting and re-establishing sessions and queues""" if self.connection.opened(): try: self.connection.close() except qpid.messaging.exceptions.ConnectionError: pass while True: try: self.connection.open() except qpid.messaging.exceptions.ConnectionError, e: LOG.error(_('Unable to connect to AMQP server: %s ' % str(e))) time.sleep(FLAGS.qpid_reconnect_interval or 1) else: break
def msg_reply(msg_id, reply=None, failure=None, ending=False): """Sends a reply or an error on the channel signified by msg_id. Failure should be a sys.exc_info() tuple. """ with ConnectionContext() as conn: if failure: message = str(failure[1]) tb = traceback.format_exception(*failure) LOG.error(_("Returning exception %s to caller"), message) LOG.error(tb) failure = (failure[0].__name__, str(failure[1]), tb) try: msg = {"result": reply, "failure": failure} except TypeError: msg = {"result": dict((k, repr(v)) for k, v in reply.__dict__.iteritems()), "failure": failure} if ending: msg["ending"] = True conn.direct_send(msg_id, msg)
def __init__(self, *args, **kwargs): for i in xrange(FLAGS.rabbit_max_retries): if i > 0: time.sleep(FLAGS.rabbit_retry_interval) try: super(Consumer, self).__init__(*args, **kwargs) self.failed_connection = False break except Exception as e: # Catching all because carrot sucks fl_host = FLAGS.rabbit_host fl_port = FLAGS.rabbit_port fl_intv = FLAGS.rabbit_retry_interval LOG.error(_('AMQP server on %(fl_host)s:%(fl_port)d is' ' unreachable: %(e)s. Trying again in %(fl_intv)d' ' seconds.') % locals()) self.failed_connection = True if self.failed_connection: LOG.error(_('Unable to connect to AMQP server ' 'after %d tries. Shutting down.'), FLAGS.rabbit_max_retries) sys.exit(1)
def __init__(self, *args, **kwargs): for i in xrange(FLAGS.rabbit_max_retries): if i > 0: time.sleep(FLAGS.rabbit_retry_interval) try: super(Consumer, self).__init__(*args, **kwargs) self.failed_connection = False break except Exception as e: # Catching all because carrot sucks fl_host = FLAGS.rabbit_host fl_port = FLAGS.rabbit_port fl_intv = FLAGS.rabbit_retry_interval LOG.error( _('AMQP server on %(fl_host)s:%(fl_port)d is' ' unreachable: %(e)s. Trying again in %(fl_intv)d' ' seconds.') % locals()) self.failed_connection = True if self.failed_connection: LOG.error( _('Unable to connect to AMQP server ' 'after %d tries. Shutting down.'), FLAGS.rabbit_max_retries) sys.exit(1)
def _connect_error(exc): log_info = {'topic': topic, 'err_str': str(exc)} LOG.error(_("Failed to declare consumer for topic '%(topic)s': " "%(err_str)s") % log_info)
def _connect_error(exc): log_info = {'topic': topic, 'err_str': str(exc)} LOG.error( _("Failed to declare consumer for topic '%(topic)s': " "%(err_str)s") % log_info)