def run(self):
# Connect to SQS input queue
self.sqs = sqs_connect(self.aws_region, **self.aws_cred)
self.sqs_input_queue = self.sqs.get_queue(self.sqs_input_queue_name)
self.sqs_input_queue.set_message_class(JSONMessage)
while True:
msgs = self.sqs_input_queue.get_messages(
num_messages = 1,
wait_time_seconds = 19,
attributes = ['ApproximateReceiveCount']
)
print "Fetched %i messages" % len(msgs)
if len(msgs) > 0:
try:
self.activeCount.increment()
msg = msgs[0]
retries = int(msg.attributes['ApproximateReceiveCount'])
if retries >= 5:
self.abort_message(msg)
print "Aborted message after %s retries, msg-id: %s" % (retries, msg['id'])
else:
self.process_message(msg)
print "Message processed successfully, msg-id: %s" % msg['id']
self.activeCount.decrement()
except:
raise
finally:
self.activeCount.decrement()
else:
sleep(23)
def process_queue(self):
# connect to sqs
sqs = sqs_connect(self.region, **self.aws_cred)
queue = sqs.get_queue(self.input_queue_name)
queue.set_message_class(JSONMessage)
# messages processed since last flush
processed_msgblocks = []
last_flush = datetime.utcnow()
while True:
print "### Handling messages"
# get new_messages from sqs
messages = []
for i in xrange(0, MESSAGE_BLOCKS_TO_MERGE):
msgs = queue.get_messages(num_messages=MESSAGE_BLOCK_SIZE)
messages += msgs
if len(msgs) > 0:
processed_msgblocks.append(msgs)
else:
break
print " - Fetched %i messages" % len(messages)
# merge messages into data folder
if len(messages) > 0:
self.merge_messages(messages)
else:
sleep(IDLE_WAIT)
# Publish if necessary
publish = False
if len(processed_msgblocks) > MESSAGE_BLOCKS_BEFORE_PUBLISH:
publish = True
if (datetime.utcnow() - last_flush).seconds > TIME_TO_PUBLISH:
publish = True
if publish:
# Skip publishing if there are no new results
if len(processed_msgblocks) == 0:
continue
self.publish_results()
# delete messages
for block in processed_msgblocks:
queue.delete_message_batch(block)
processed_msgblocks = []
last_flush = datetime.utcnow()
def process_queue(self):
# connect to sqs
sqs = sqs_connect(self.region, **self.aws_cred)
queue = sqs.get_queue(self.input_queue_name)
queue.set_message_class(JSONMessage)
# messages processed since last flush
processed_msgblocks = []
last_flush = datetime.utcnow()
while True:
print "### Handling messages"
# get new_messages from sqs
messages = []
for i in xrange(0, MESSAGE_BLOCKS_TO_MERGE):
msgs = queue.get_messages(num_messages = MESSAGE_BLOCK_SIZE)
messages += msgs
if len(msgs) > 0:
processed_msgblocks.append(msgs)
else:
break
print " - Fetched %i messages" % len(messages)
# merge messages into data folder
if len(messages) > 0:
self.merge_messages(messages)
else:
sleep(IDLE_WAIT)
# Publish if necessary
publish = False
if len(processed_msgblocks) > MESSAGE_BLOCKS_BEFORE_PUBLISH:
publish = True
if (datetime.utcnow() - last_flush).seconds > TIME_TO_PUBLISH:
publish = True
if publish:
# Skip publishing if there are no new results
if len(processed_msgblocks) == 0:
continue
self.publish_results()
# delete messages
for block in processed_msgblocks:
queue.delete_message_batch(block)
processed_msgblocks = []
last_flush = datetime.utcnow()