def run(self): '''Runs the pipeline step. ''' queue = QueueClient(account_url=os.getenv('AZ_QS_AC_URL'), queue_name=os.getenv('AZ_QS_QUEUE_NAME'), credential=os.getenv('AZ_QS_SAS_TOKEN')) response = queue.receive_messages(messages_per_page=5) for batch in response.by_page(): for message in batch: filename = message.content label, _ = classifier.predict(filename) self.print('\'{filename}\' classified as \'{label}\'', filename=filename, label=label['name']) with open(self.output['classifier'], 'a') as f: f.write(message.content + ',' + str(label['id']) + '\n') if label['name'] != 'other': entities = ner.predict(filename) self.print('\'{filename}\' has entities \'{entities}\'', filename=filename, entities=entities) with open(self.output['ner'], 'a') as f: f.write(message.content + ',\"' + str(entities) + '\"\n') queue.delete_message(message)
def test_sas_process(self, resource_group, location, storage_account, storage_account_key): # SAS URL is calculated from storage key, so this test runs live only if not self.is_live: return # Arrange qsc = QueueServiceClient(self._account_url(storage_account.name), storage_account_key) queue_client = self._get_queue_reference(qsc) queue_client.create_queue() queue_client.enqueue_message(u'message1') token = queue_client.generate_shared_access_signature( QueueSasPermissions(process=True), datetime.utcnow() + timedelta(hours=1), ) # Act service = QueueClient( queue_url=queue_client.url, credential=token, ) message = next(service.receive_messages()) # Assert self.assertIsNotNone(message) self.assertNotEqual('', message.id) self.assertEqual(u'message1', message.content)
def test_set_access_policy(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # [START create_queue_client_from_connection_string] from azure.storage.queue import QueueClient queue_client = QueueClient.from_connection_string(self.connection_string, "queuetest") # [END create_queue_client_from_connection_string] # Create the queue try: queue_client.create_queue() except ResourceExistsError: pass queue_client.enqueue_message(u"hello world") try: # [START set_access_policy] # Create an access policy from azure.storage.queue import AccessPolicy, QueuePermissions access_policy = AccessPolicy() access_policy.start = datetime.utcnow() - timedelta(hours=1) access_policy.expiry = datetime.utcnow() + timedelta(hours=1) access_policy.permission = QueuePermissions.READ identifiers = {'my-access-policy-id': access_policy} # Set the access policy queue_client.set_queue_access_policy(identifiers) # [END set_access_policy] # Use the access policy to generate a SAS token # [START queue_client_sas_token] sas_token = queue_client.generate_shared_access_signature( policy_id='my-access-policy-id' ) # [END queue_client_sas_token] # Authenticate with the sas token # [START create_queue_client] q = QueueClient( queue_url=queue_client.url, credential=sas_token ) # [END create_queue_client] # Use the newly authenticated client to receive messages my_message = q.receive_messages() assert my_message is not None finally: # Delete the queue queue_client.delete_queue()
def main(accountName, queueName): accountURL = "https://%s.queue.core.windows.net" % (accountName) creds = ManagedIdentityCredential() client = QueueClient(account_url=accountURL, queue_name=queueName, credential=creds) messages = client.receive_messages(messages_per_page=1) for message in messages: print(message.content) client.delete_message(message)
def _pop_specific_q(_q: QueueClient, _n: int) -> bool: has_messages = False for m in _q.receive_messages(messages_per_page=_n): if m: has_messages = True result.append(m if raw else self._deserialize_message(m)) if delete: _q.delete_message(m.id, m.pop_receipt) # short circuit to prevent unneeded work if len(result) == n: return True return has_messages
def test_sas_process(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # Arrange queue_client = self._create_queue() queue_client.enqueue_message(u'message1') token = queue_client.generate_shared_access_signature( QueuePermissions.PROCESS, datetime.utcnow() + timedelta(hours=1), ) # Act service = QueueClient( queue_url=queue_client.url, credential=token, ) message = next(service.receive_messages()) # Assert self.assertIsNotNone(message) self.assertNotEqual('', message.id) self.assertEqual(u'message1', message.content)
str(metadata.approximate_message_count)) time.sleep(1) ### # Use the Azure Storage Storage SDK for Python to read each message from the Queue ### print( '\nWith some messages in our Azure Storage Queue, let\'s read the first message in the Queue to signal we start to process that customer\'s order.' ) raw_input('Press Enter to continue...') # When you get each message, they become hidden from other parts of the applications being able to see it. # Once you have successfully processed the message, you then delete the message from the Queue. # This behavior makes sure that if something goes wrong in the processing of the message, it is then dropped back in the Queue for processing in the next cycle. messages = queue_client.receive_messages() for message in messages: print('\n' + message.content) queue_client.delete_message(message.id, message.pop_receipt) raw_input('\nPress Enter to continue...') metadata = queue_client.get_queue_properties() print( 'If we look at the Queue again, we have one less message to show we have processed that order and a yummy pizza will be on it\'s way to the customer soon.' ) print('Number of messages in the queue: ' + str(metadata.approximate_message_count)) raw_input('\nPress Enter to continue...') ###