def run(self, parallel=False): if parallel: self.logger.info( "Parallel processing with Azure Queue is not yet implemented.") self.logger.info("Downloading messages from the Azure Storage queue.") queue_settings = StorageUtilities.get_queue_client_by_uri( self.receive_queue) queue_messages = StorageUtilities.get_queue_messages( *queue_settings, num_messages=self.batch_size) while len(queue_messages) > 0: for queue_message in queue_messages: self.logger.debug("Message id: %s received" % queue_message.id) if (self.process_azure_queue_message(queue_message) or queue_message.dequeue_count > self.max_message_retry): # If message handled successfully or max retry hit, delete StorageUtilities.delete_queue_message( *queue_settings, message=queue_message) queue_messages = StorageUtilities.get_queue_messages( *queue_settings, num_messages=self.batch_size) self.logger.info( 'No messages left on the azure storage queue, exiting c7n_mailer.')
def test_notify_though_storage_queue(self): account = self.setup_account() # Create queue, make sure it is empty queue_url = "https://" + account.name + ".queue.core.windows.net/testnotify" queue, name = StorageUtilities.get_queue_client_by_uri( queue_url, self.session) queue.clear_messages(name) p = self.load_policy({ 'name': 'test-notify-for-keyvault', 'resource': 'azure.keyvault', 'actions': [{ 'type': 'notify', 'template': 'default', 'priority_header': '2', 'subject': 'testing notify action', 'to': ['*****@*****.**'], 'transport': { 'type': 'asq', 'queue': queue_url } }] }) resources = p.run() self.assertEqual(len(resources), 1) # Pull messages, should be 1 messages = StorageUtilities.get_queue_messages(queue, name) self.assertEqual(len(messages), 1)
def test_notify_though_storage_queue(self): account = self.setup_account() # Create queue, make sure it is empty queue_url = "https://" + account.name + ".queue.core.windows.net/testnotify" queue, name = StorageUtilities.get_queue_client_by_uri(queue_url, self.session) queue.clear_messages(name) p = self.load_policy({ 'name': 'test-notify-for-keyvault', 'resource': 'azure.keyvault', 'actions': [ {'type': 'notify', 'template': 'default', 'priority_header': '2', 'subject': 'testing notify action', 'to': ['*****@*****.**'], 'transport': {'type': 'asq', 'queue': queue_url} } ] }) resources = p.run() self.assertEqual(len(resources), 1) # Pull messages, should be 1 messages = StorageUtilities.get_queue_messages(queue, name) self.assertEqual(len(messages), 1)
def test_cycle_queue_message_by_uri(self): account = self.setup_account() url = "https://" + account.name + ".queue.core.windows.net/testcyclemessage" queue_settings = StorageUtilities.get_queue_client_by_uri(url, self.session) StorageUtilities.put_queue_message(*queue_settings, content=u"hello queue") # Pull messages, should be 1 messages = StorageUtilities.get_queue_messages(*queue_settings) self.assertEqual(len(messages), 1) # Read message and delete it from queue for message in messages: self.assertEqual(message.content, u"hello queue") StorageUtilities.delete_queue_message(*queue_settings, message=message) # Pull messages again, should be empty messages = StorageUtilities.get_queue_messages(*queue_settings) self.assertEqual(len(messages), 0)
def run(self, parallel=False): if parallel: self.logger.info("Parallel processing with Azure Queue is not yet implemented.") self.logger.info("Downloading messages from the Azure Storage queue.") queue_settings = StorageUtilities.get_queue_client_by_uri(self.receive_queue, self.session) queue_messages = StorageUtilities.get_queue_messages( *queue_settings, num_messages=self.batch_size) while len(queue_messages) > 0: for queue_message in queue_messages: self.logger.debug("Message id: %s received" % queue_message.id) if (self.process_azure_queue_message(queue_message) or queue_message.dequeue_count > self.max_message_retry): # If message handled successfully or max retry hit, delete StorageUtilities.delete_queue_message(*queue_settings, message=queue_message) queue_messages = StorageUtilities.get_queue_messages( *queue_settings, num_messages=self.batch_size) self.logger.info('No messages left on the azure storage queue, exiting c7n_mailer.')
def poll_queue(self): """ Poll the Azure queue and loop until there are no visible messages remaining. """ # Exit if we don't have any policies if not self.policies: return if not self.queue_service: self.queue_service = Storage.get_queue_client_by_storage_account( self.queue_storage_account, self.storage_session) while True: try: messages = Storage.get_queue_messages( self.queue_service, self.event_queue_name, num_messages=queue_message_count, visibility_timeout=queue_timeout_seconds) except AzureHttpError: self.queue_service = None raise if len(messages) == 0: break log.info('Pulled %s events to process while polling queue.' % len(messages)) for message in messages: if message.dequeue_count > max_dequeue_count: Storage.delete_queue_message(self.queue_service, self.event_queue_name, message=message) log.warning("Event deleted due to reaching maximum retry count.") else: # Run matching policies self.run_policies_for_event(message) # We delete events regardless of policy result Storage.delete_queue_message( self.queue_service, self.event_queue_name, message=message)