def loop(self):
poll_delay = float(conf.get("rabbit_broker", "poll_delay"))
update_timer = int(conf.get("global", "update_timer"))
start_time = datetime.datetime.now()
messages_sent = {}
while True:
try:
for consumer in self.consumers:
messages = []
if not consumer.queue_name in messages_sent:
messages_sent[consumer.queue_name] = 0
for n in xrange(consumer.max_messages):
msg = consumer.consumer.fetch(enable_callbacks=False)
if not msg:
break
LOG.debug("Received message on queue %s" %
consumer.queue_name)
messages.append(msg)
num_messages = len(messages)
if num_messages > 0:
consumer.fetched_messages(messages)
messages_sent[consumer.queue_name] += num_messages
# Ingnoring microseconds because we're not going to let you
# be that granular and it's not super useful anyway
td = datetime.datetime.now() - start_time
elapsed = td.seconds + td.days * 86400
if elapsed > update_timer:
# This isn't threaded, it's just best effort
LOG.info("Update timer elapsed: %s seconds", str(elapsed))
total_messages = 0
for consumer in self.consumers:
LOG.info("\tSent %d messages from %s" %
(messages_sent[consumer.queue_name],
consumer.queue_name))
total_messages += messages_sent[consumer.queue_name]
LOG.info("\tSent %d total messages" % total_messages)
if total_messages > 0:
LOG.info("\tMessages per second: %f" %
(float(total_messages) / elapsed))
start_time = datetime.datetime.now()
messages_sent = {}
# This should really only be used when trying to discern bugs
# in the flood of messages and coupled with DEBUG logging.
# Otherwise, we want Yagi sending as quickly as possible
if poll_delay:
time.sleep(poll_delay)
except socket.error, e:
LOG.critical("Rabbit connection lost, reconnecting")
LOG.exception(e)
self.establish_consumer_connection(consumer)
except amqplib.client_0_8.exceptions.AMQPException, e:
LOG.critical("Rabbit connection lost, reconnecting")
LOG.exception(e)
self.establish_consumer_connection(consumer)
def loop(self):
poll_delay = float(conf.get('rabbit_broker', 'poll_delay'))
while True:
try:
for consumer in self.consumers:
messages = []
for n in xrange(consumer.max_messages()):
msg = consumer.consumer.fetch(enable_callbacks=False)
if not msg:
break
try:
LOG.info('Received message on queue %s' %
consumer.queue_name)
messages.append(msg.payload)
except Exception, e:
LOG.error('Message decoding failed!')
continue
if not msg.acknowledged:
msg.ack()
consumer.fetched_messages(messages)
if poll_delay:
time.sleep(poll_delay)
except socket.error, e:
LOG.critical("Rabbit connection lost, reconnecting")
self.establish_consumer_connection(consumer)
def loop(self):
poll_delay = float(conf.get("rabbit_broker", "poll_delay"))
update_timer = int(conf.get("global", "update_timer"))
max_connection_age = int(
conf.get("rabbit_broker", "max_connection_age"))
start_time = datetime.datetime.now()
messages_sent = {}
while True:
try:
for consumer in self.consumers:
messages = []
if not consumer.queue_name in messages_sent:
messages_sent[consumer.queue_name] = 0
for n in xrange(consumer.max_messages):
msg = consumer.consumer.fetch(enable_callbacks=False)
if not msg:
break
LOG.debug("Received message on queue %s" %
consumer.queue_name)
messages.append(msg)
num_messages = len(messages)
if num_messages > 0:
consumer.fetched_messages(messages)
messages_sent[consumer.queue_name] += num_messages
if max_connection_age > 0:
age = datetime.datetime.now() - consumer.connect_time
age_sec = age.seconds + (age.days * 86400)
if age_sec > max_connection_age:
LOG.info("Maximum AMQP connection time for "
"connection to %s reached. "
"Reconnecting..." % consumer.queue_name)
self.establish_consumer_connection(consumer)
# Ingnoring microseconds because we're not going to let you
# be that granular and it's not super useful anyway
td = datetime.datetime.now() - start_time
elapsed = td.seconds + td.days * 86400
if elapsed > update_timer:
# This isn't threaded, it's just best effort
LOG.info("Update timer elapsed: %s seconds", str(elapsed))
total_messages = 0
for consumer in self.consumers:
LOG.info("\tSent %d messages from %s" %
(messages_sent[consumer.queue_name],
consumer.queue_name))
total_messages += messages_sent[consumer.queue_name]
LOG.info("\tSent %d total messages" % total_messages)
if total_messages > 0:
LOG.info("\tMessages per second: %f" %
(float(total_messages) / elapsed))
start_time = datetime.datetime.now()
# Periodically call the consumers in case they need
# to do some work even when there are no events to
# process ...
for consumer in self.consumers:
consumer.idle(messages_sent[consumer.queue_name],
consumer.queue_name)
messages_sent = {}
# This should really only be used when trying to discern bugs
# in the flood of messages and coupled with DEBUG logging.
# Otherwise, we want Yagi sending as quickly as possible
if poll_delay:
time.sleep(poll_delay)
except socket.error, e:
LOG.critical("Rabbit connection lost, reconnecting")
LOG.exception(e)
self.establish_consumer_connection(consumer)
except amqplib.client_0_8.exceptions.AMQPException, e:
LOG.critical("Rabbit connection lost, reconnecting")
LOG.exception(e)
self.establish_consumer_connection(consumer)
