def run_forever(self):
"""Keep running (unless we get a Ctrl-C).
"""
while True:
message = None
try:
self.wait_if_paused()
message = self.source_queue.get(block=True)
data = message.payload
with stats.time_block(self.statsd_client):
new_messages = self.handle_data(data)
# Must be successful if we have reached here.
stats.mark_successful_job(self.statsd_client)
self.post_handle_data()
except KeyboardInterrupt:
logging.info("Caught Ctrl-C. Byee!")
# Break out of our loop.
#
break
except:
# Keep going, but don't ack the message.
# Also, log the exception.
logging.exception("Exception handling data")
if self.handle_exception is not None:
self.handle_exception()
if message:
if self.requeue:
message.requeue()
elif self.reject:
message.reject()
stats.mark_failed_job(self.statsd_client)
else:
# Queue up the new messages (if any).
#
if new_messages:
self.queue_new_messages(new_messages)
# We're done with the original message.
#
message.ack()
def batched_run_forever(self, size, wait_timeout_seconds=5):
"""This will take messages off the queue and put them in a buffer.
Once the buffer reaches the given size, handle_data is called for the
entire buffer. (So handle_data must be able to handle a list.)
If handle_data doesn't throw an exception, all messages are ack'd.
Otherwise all messages are requeued/rejected.
"""
buffer = []
while True:
new_messages = []
try:
self.wait_if_paused()
queue_was_empty = False
message = None
try:
# We need to have a timeout. Otherwise if we had no more
# messages coming in, but len(buffer) < size, then the
# buffer would never get processed!
message = self.source_queue.get(block=True, timeout=wait_timeout_seconds)
except queue.Empty:
queue_was_empty = True
if message:
buffer.append(message)
# We proceed to handle the buffer if
# 1. it has reached the given size or
# 2. we drained the queue, but the buffer is smaller than size
if len(buffer) >= size or (buffer and queue_was_empty):
try:
with stats.time_block(self.statsd_client):
new_messages = self.handle_batch(buffer)
stats.mark_successful_job(self.statsd_client)
self.post_handle_data()
except KeyboardInterrupt as ki:
# Raise this for the outer try to handle.
raise ki
except:
logging.exception("Exception handling batch")
if self.handle_exception is not None:
self.handle_exception()
stats.mark_failed_job(self.statsd_client)
finally:
# If all went well then we have ack'd the messages.
# If not, we have requeued or rejected them.
# Either way we are done with the buffer.
buffer = []
except KeyboardInterrupt:
logging.info("Caught Ctrl-C. Byee!")
break
except:
# When could this happen? Perhaps while waiting?
logging.exception("Exception elsewhere in batched_run_forever")
else:
if new_messages:
self.queue_new_messages(new_messages)