def run(self, parallel=False):
if parallel:
self.logger.info(
"Parallel processing with Azure Queue is not yet implemented.")
self.logger.info("Downloading messages from the Azure Storage queue.")
queue_settings = StorageUtilities.get_queue_client_by_uri(
self.receive_queue)
queue_messages = StorageUtilities.get_queue_messages(
*queue_settings, num_messages=self.batch_size)
while len(queue_messages) > 0:
for queue_message in queue_messages:
self.logger.debug("Message id: %s received" % queue_message.id)
if (self.process_azure_queue_message(queue_message) or
queue_message.dequeue_count > self.max_message_retry):
# If message handled successfully or max retry hit, delete
StorageUtilities.delete_queue_message(
*queue_settings, message=queue_message)
queue_messages = StorageUtilities.get_queue_messages(
*queue_settings, num_messages=self.batch_size)
self.logger.info(
'No messages left on the azure storage queue, exiting c7n_mailer.')
def poll_queue(self):
"""
Poll the Azure queue and loop until
there are no visible messages remaining.
"""
# Exit if we don't have any policies
if not self.policies:
return
if not self.queue_service:
self.queue_service = Storage.get_queue_client_by_storage_account(
self.queue_storage_account,
self.storage_session)
while True:
try:
messages = Storage.get_queue_messages(
self.queue_service,
self.event_queue_name,
num_messages=queue_message_count,
visibility_timeout=queue_timeout_seconds)
except AzureHttpError:
self.queue_service = None
raise
if len(messages) == 0:
break
log.info('Pulled %s events to process while polling queue.' % len(messages))
for message in messages:
if message.dequeue_count > max_dequeue_count:
Storage.delete_queue_message(self.queue_service,
self.event_queue_name,
message=message)
log.warning("Event deleted due to reaching maximum retry count.")
else:
# Run matching policies
self.run_policies_for_event(message)
# We delete events regardless of policy result
Storage.delete_queue_message(
self.queue_service,
self.event_queue_name,
message=message)
def test_cycle_queue_message_by_uri(self):
account = self.setup_account()
url = "https://" + account.name + ".queue.core.windows.net/testcyclemessage"
queue_settings = StorageUtilities.get_queue_client_by_uri(url, self.session)
StorageUtilities.put_queue_message(*queue_settings, content=u"hello queue")
# Pull messages, should be 1
messages = StorageUtilities.get_queue_messages(*queue_settings)
self.assertEqual(len(messages), 1)
# Read message and delete it from queue
for message in messages:
self.assertEqual(message.content, u"hello queue")
StorageUtilities.delete_queue_message(*queue_settings, message=message)
# Pull messages again, should be empty
messages = StorageUtilities.get_queue_messages(*queue_settings)
self.assertEqual(len(messages), 0)
def run(self, parallel=False):
if parallel:
self.logger.info("Parallel processing with Azure Queue is not yet implemented.")
self.logger.info("Downloading messages from the Azure Storage queue.")
queue_settings = StorageUtilities.get_queue_client_by_uri(self.receive_queue, self.session)
queue_messages = StorageUtilities.get_queue_messages(
*queue_settings, num_messages=self.batch_size)
while len(queue_messages) > 0:
for queue_message in queue_messages:
self.logger.debug("Message id: %s received" % queue_message.id)
if (self.process_azure_queue_message(queue_message) or
queue_message.dequeue_count > self.max_message_retry):
# If message handled successfully or max retry hit, delete
StorageUtilities.delete_queue_message(*queue_settings, message=queue_message)
queue_messages = StorageUtilities.get_queue_messages(
*queue_settings, num_messages=self.batch_size)
self.logger.info('No messages left on the azure storage queue, exiting c7n_mailer.')
