def loop(self): poll_delay = float(conf.get("rabbit_broker", "poll_delay")) update_timer = int(conf.get("global", "update_timer")) start_time = datetime.datetime.now() messages_sent = {} while True: try: for consumer in self.consumers: messages = [] if not consumer.queue_name in messages_sent: messages_sent[consumer.queue_name] = 0 for n in xrange(consumer.max_messages): msg = consumer.consumer.fetch(enable_callbacks=False) if not msg: break LOG.debug("Received message on queue %s" % consumer.queue_name) messages.append(msg) num_messages = len(messages) if num_messages > 0: consumer.fetched_messages(messages) messages_sent[consumer.queue_name] += num_messages # Ingnoring microseconds because we're not going to let you # be that granular and it's not super useful anyway td = datetime.datetime.now() - start_time elapsed = td.seconds + td.days * 86400 if elapsed > update_timer: # This isn't threaded, it's just best effort LOG.info("Update timer elapsed: %s seconds", str(elapsed)) total_messages = 0 for consumer in self.consumers: LOG.info("\tSent %d messages from %s" % (messages_sent[consumer.queue_name], consumer.queue_name)) total_messages += messages_sent[consumer.queue_name] LOG.info("\tSent %d total messages" % total_messages) if total_messages > 0: LOG.info("\tMessages per second: %f" % (float(total_messages) / elapsed)) start_time = datetime.datetime.now() messages_sent = {} # This should really only be used when trying to discern bugs # in the flood of messages and coupled with DEBUG logging. # Otherwise, we want Yagi sending as quickly as possible if poll_delay: time.sleep(poll_delay) except socket.error, e: LOG.critical("Rabbit connection lost, reconnecting") LOG.exception(e) self.establish_consumer_connection(consumer) except amqplib.client_0_8.exceptions.AMQPException, e: LOG.critical("Rabbit connection lost, reconnecting") LOG.exception(e) self.establish_consumer_connection(consumer)
def loop(self): poll_delay = float(conf.get('rabbit_broker', 'poll_delay')) while True: try: for consumer in self.consumers: messages = [] for n in xrange(consumer.max_messages()): msg = consumer.consumer.fetch(enable_callbacks=False) if not msg: break try: LOG.info('Received message on queue %s' % consumer.queue_name) messages.append(msg.payload) except Exception, e: LOG.error('Message decoding failed!') continue if not msg.acknowledged: msg.ack() consumer.fetched_messages(messages) if poll_delay: time.sleep(poll_delay) except socket.error, e: LOG.critical("Rabbit connection lost, reconnecting") self.establish_consumer_connection(consumer)
def loop(self): poll_delay = float(conf.get("rabbit_broker", "poll_delay")) update_timer = int(conf.get("global", "update_timer")) max_connection_age = int( conf.get("rabbit_broker", "max_connection_age")) start_time = datetime.datetime.now() messages_sent = {} while True: try: for consumer in self.consumers: messages = [] if not consumer.queue_name in messages_sent: messages_sent[consumer.queue_name] = 0 for n in xrange(consumer.max_messages): msg = consumer.consumer.fetch(enable_callbacks=False) if not msg: break LOG.debug("Received message on queue %s" % consumer.queue_name) messages.append(msg) num_messages = len(messages) if num_messages > 0: consumer.fetched_messages(messages) messages_sent[consumer.queue_name] += num_messages if max_connection_age > 0: age = datetime.datetime.now() - consumer.connect_time age_sec = age.seconds + (age.days * 86400) if age_sec > max_connection_age: LOG.info("Maximum AMQP connection time for " "connection to %s reached. " "Reconnecting..." % consumer.queue_name) self.establish_consumer_connection(consumer) # Ingnoring microseconds because we're not going to let you # be that granular and it's not super useful anyway td = datetime.datetime.now() - start_time elapsed = td.seconds + td.days * 86400 if elapsed > update_timer: # This isn't threaded, it's just best effort LOG.info("Update timer elapsed: %s seconds", str(elapsed)) total_messages = 0 for consumer in self.consumers: LOG.info("\tSent %d messages from %s" % (messages_sent[consumer.queue_name], consumer.queue_name)) total_messages += messages_sent[consumer.queue_name] LOG.info("\tSent %d total messages" % total_messages) if total_messages > 0: LOG.info("\tMessages per second: %f" % (float(total_messages) / elapsed)) start_time = datetime.datetime.now() # Periodically call the consumers in case they need # to do some work even when there are no events to # process ... for consumer in self.consumers: consumer.idle(messages_sent[consumer.queue_name], consumer.queue_name) messages_sent = {} # This should really only be used when trying to discern bugs # in the flood of messages and coupled with DEBUG logging. # Otherwise, we want Yagi sending as quickly as possible if poll_delay: time.sleep(poll_delay) except socket.error, e: LOG.critical("Rabbit connection lost, reconnecting") LOG.exception(e) self.establish_consumer_connection(consumer) except amqplib.client_0_8.exceptions.AMQPException, e: LOG.critical("Rabbit connection lost, reconnecting") LOG.exception(e) self.establish_consumer_connection(consumer)