def test_sas_add(self, resource_group, location, storage_account, storage_account_key): # SAS URL is calculated from storage key, so this test runs live only if not self.is_live: return # Arrange qsc = QueueServiceClient(self._account_url(storage_account.name), storage_account_key) queue_client = self._get_queue_reference(qsc) queue_client.create_queue() token = queue_client.generate_shared_access_signature( QueuePermissions.ADD, datetime.utcnow() + timedelta(hours=1), ) # Act service = QueueClient( queue_url=queue_client.url, credential=token, ) result = service.enqueue_message(u'addedmessage') # Assert result = next(queue_client.receive_messages()) self.assertEqual(u'addedmessage', result.content)
def test_sas_process(self, resource_group, location, storage_account, storage_account_key): # SAS URL is calculated from storage key, so this test runs live only if not self.is_live: return # Arrange qsc = QueueServiceClient(self._account_url(storage_account.name), storage_account_key) queue_client = self._get_queue_reference(qsc) queue_client.create_queue() queue_client.enqueue_message(u'message1') token = queue_client.generate_shared_access_signature( QueueSasPermissions(process=True), datetime.utcnow() + timedelta(hours=1), ) # Act service = QueueClient( queue_url=queue_client.url, credential=token, ) message = next(service.receive_messages()) # Assert self.assertIsNotNone(message) self.assertNotEqual('', message.id) self.assertEqual(u'message1', message.content)
def test_sas_read(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # Arrange queue_client = self._create_queue() queue_client.enqueue_message(u'message1') token = queue_client.generate_shared_access_signature( QueuePermissions.READ, datetime.utcnow() + timedelta(hours=1), datetime.utcnow() - timedelta(minutes=5)) # Act service = QueueClient( queue_url=queue_client.url, credential=token, ) result = service.peek_messages() # Assert self.assertIsNotNone(result) self.assertEqual(1, len(result)) message = result[0] self.assertIsNotNone(message) self.assertNotEqual('', message.id) self.assertEqual(u'message1', message.content)
def test_sas_update(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # Arrange queue_client = self._create_queue() queue_client.enqueue_message(u'message1') token = queue_client.generate_shared_access_signature( QueuePermissions.UPDATE, datetime.utcnow() + timedelta(hours=1), ) messages = queue_client.receive_messages() result = next(messages) # Act service = QueueClient( queue_url=queue_client.url, credential=token, ) service.update_message( result.id, pop_receipt=result.pop_receipt, visibility_timeout=0, content=u'updatedmessage1', ) # Assert result = next(messages) self.assertEqual(u'updatedmessage1', result.content)
def test_sas_update(self, resource_group, location, storage_account, storage_account_key): # SAS URL is calculated from storage key, so this test runs live only if not self.is_live: return # Arrange qsc = QueueServiceClient(self._account_url(storage_account.name), storage_account_key) queue_client = self._get_queue_reference(qsc) queue_client.create_queue() queue_client.enqueue_message(u'message1') token = queue_client.generate_shared_access_signature( QueueSasPermissions(update=True), datetime.utcnow() + timedelta(hours=1), ) messages = queue_client.receive_messages() result = next(messages) # Act service = QueueClient( queue_url=queue_client.url, credential=token, ) service.update_message( result.id, pop_receipt=result.pop_receipt, visibility_timeout=0, content=u'updatedmessage1', ) # Assert result = next(messages) self.assertEqual(u'updatedmessage1', result.content)
def test_sas_signed_identifier(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # Arrange access_policy = AccessPolicy() access_policy.start = datetime.utcnow() - timedelta(hours=1) access_policy.expiry = datetime.utcnow() + timedelta(hours=1) access_policy.permission = QueuePermissions.READ identifiers = {'testid': access_policy} queue_client = self._create_queue() resp = queue_client.set_queue_access_policy(identifiers) queue_client.enqueue_message(u'message1') token = queue_client.generate_shared_access_signature( policy_id='testid') # Act service = QueueClient( queue_url=queue_client.url, credential=token, ) result = service.peek_messages() # Assert self.assertIsNotNone(result) self.assertEqual(1, len(result)) message = result[0] self.assertIsNotNone(message) self.assertNotEqual('', message.id) self.assertEqual(u'message1', message.content)
def test_create_service_with_custom_account_endpoint_path(self, resource_group, location, storage_account, storage_account_key): custom_account_url = "http://local-machine:11002/custom/account/path/" + self.sas_token for service_type in SERVICES.items(): conn_string = 'DefaultEndpointsProtocol=http;AccountName={};AccountKey={};QueueEndpoint={};'.format( storage_account.name, storage_account_key, custom_account_url) # Act service = service_type[0].from_connection_string(conn_string, queue_name="foo") # Assert self.assertEqual(service.account_name, storage_account.name) self.assertEqual(service.credential.account_name, storage_account.name) self.assertEqual(service.credential.account_key, storage_account_key) self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path') service = QueueServiceClient(account_url=custom_account_url) self.assertEqual(service.account_name, None) self.assertEqual(service.credential, None) self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path') self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/?')) service = QueueClient(account_url=custom_account_url, queue_name="foo") self.assertEqual(service.account_name, None) self.assertEqual(service.queue_name, "foo") self.assertEqual(service.credential, None) self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path') self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/foo?')) service = QueueClient.from_queue_url("http://local-machine:11002/custom/account/path/foo" + self.sas_token) self.assertEqual(service.account_name, None) self.assertEqual(service.queue_name, "foo") self.assertEqual(service.credential, None) self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path') self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/foo?'))
def run(self): '''Runs the pipeline step. ''' queue = QueueClient(account_url=os.getenv('AZ_QS_AC_URL'), queue_name=os.getenv('AZ_QS_QUEUE_NAME'), credential=os.getenv('AZ_QS_SAS_TOKEN')) response = queue.receive_messages(messages_per_page=5) for batch in response.by_page(): for message in batch: filename = message.content label, _ = classifier.predict(filename) self.print('\'{filename}\' classified as \'{label}\'', filename=filename, label=label['name']) with open(self.output['classifier'], 'a') as f: f.write(message.content + ',' + str(label['id']) + '\n') if label['name'] != 'other': entities = ner.predict(filename) self.print('\'{filename}\' has entities \'{entities}\'', filename=filename, entities=entities) with open(self.output['ner'], 'a') as f: f.write(message.content + ',\"' + str(entities) + '\"\n') queue.delete_message(message)
def test_set_access_policy(self, resource_group, location, storage_account, storage_account_key): # SAS URL is calculated from storage key, so this test runs live only if not self.is_live: return # [START create_queue_client_from_connection_string] from azure.storage.queue import QueueClient queue_client = QueueClient.from_connection_string(self.connection_string(storage_account, storage_account_key), "queuetest") # [END create_queue_client_from_connection_string] # Create the queue try: queue_client.create_queue() except ResourceExistsError: pass queue_client.send_message(u"hello world") try: # [START set_access_policy] # Create an access policy from azure.storage.queue import AccessPolicy, QueueSasPermissions access_policy = AccessPolicy() access_policy.start = datetime.utcnow() - timedelta(hours=1) access_policy.expiry = datetime.utcnow() + timedelta(hours=1) access_policy.permission = QueueSasPermissions(read=True) identifiers = {'my-access-policy-id': access_policy} # Set the access policy queue_client.set_queue_access_policy(identifiers) # [END set_access_policy] # Use the access policy to generate a SAS token # [START queue_client_sas_token] from azure.storage.queue import generate_queue_sas sas_token = generate_queue_sas( queue_client.account_name, queue_client.queue_name, queue_client.credential.account_key, policy_id='my-access-policy-id' ) # [END queue_client_sas_token] # Authenticate with the sas token # [START create_queue_client] q = QueueClient.from_queue_url( queue_url=queue_client.url, credential=sas_token ) # [END create_queue_client] # Use the newly authenticated client to receive messages my_message = q.receive_messages() assert my_message is not None finally: # Delete the queue queue_client.delete_queue()
def test_set_access_policy(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # [START create_queue_client_from_connection_string] from azure.storage.queue import QueueClient queue_client = QueueClient.from_connection_string(self.connection_string, "queuetest") # [END create_queue_client_from_connection_string] # Create the queue try: queue_client.create_queue() except ResourceExistsError: pass queue_client.enqueue_message(u"hello world") try: # [START set_access_policy] # Create an access policy from azure.storage.queue import AccessPolicy, QueuePermissions access_policy = AccessPolicy() access_policy.start = datetime.utcnow() - timedelta(hours=1) access_policy.expiry = datetime.utcnow() + timedelta(hours=1) access_policy.permission = QueuePermissions.READ identifiers = {'my-access-policy-id': access_policy} # Set the access policy queue_client.set_queue_access_policy(identifiers) # [END set_access_policy] # Use the access policy to generate a SAS token # [START queue_client_sas_token] sas_token = queue_client.generate_shared_access_signature( policy_id='my-access-policy-id' ) # [END queue_client_sas_token] # Authenticate with the sas token # [START create_queue_client] q = QueueClient( queue_url=queue_client.url, credential=sas_token ) # [END create_queue_client] # Use the newly authenticated client to receive messages my_message = q.receive_messages() assert my_message is not None finally: # Delete the queue queue_client.delete_queue()
def main(accountName, queueName): accountURL = "https://%s.queue.core.windows.net" % (accountName) creds = ManagedIdentityCredential() client = QueueClient(account_url=accountURL, queue_name=queueName, credential=creds) messages = client.receive_messages(messages_per_page=1) for message in messages: print(message.content) client.delete_message(message)
def _pop_specific_q(_q: QueueClient, _n: int) -> bool: has_messages = False for m in _q.receive_messages(messages_per_page=_n): if m: has_messages = True result.append(m if raw else self._deserialize_message(m)) if delete: _q.delete_message(m.id, m.pop_receipt) # short circuit to prevent unneeded work if len(result) == n: return True return has_messages
def set_access_policy(self): # [START create_queue_client_from_connection_string] from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string(self.connection_string, "myqueue1") # [END create_queue_client_from_connection_string] # Create the queue queue.create_queue() # Send a message queue.send_message(u"hello world") try: # [START set_access_policy] # Create an access policy from azure.storage.queue import AccessPolicy, QueueSasPermissions access_policy = AccessPolicy() access_policy.start = datetime.utcnow() - timedelta(hours=1) access_policy.expiry = datetime.utcnow() + timedelta(hours=1) access_policy.permission = QueueSasPermissions(read=True) identifiers = {'my-access-policy-id': access_policy} # Set the access policy queue.set_queue_access_policy(identifiers) # [END set_access_policy] # Use the access policy to generate a SAS token # [START queue_client_sas_token] from azure.storage.queue import generate_queue_sas sas_token = generate_queue_sas( queue.account_name, queue.queue_name, queue.credential.account_key, policy_id='my-access-policy-id' ) # [END queue_client_sas_token] # Authenticate with the sas token # [START create_queue_client] token_auth_queue = QueueClient.from_queue_url( queue_url=queue.url, credential=sas_token ) # [END create_queue_client] # Use the newly authenticated client to receive messages my_message = token_auth_queue.receive_messages() finally: # Delete the queue queue.delete_queue()
def get_queue(queue_name, create_queue, clear_queue): ''' Note that generating the queueclient does not mean there must a queue there as one of the properties of queueclient is "create_queue", so it's really a representation of a queue which may or may not exist yet. ''' keyVaultName = os.environ["KEY_VAULT_NAME"] keyVault_URI = "https://" + keyVaultName + ".vault.azure.net" credential = DefaultAzureCredential() client = SecretClient(vault_url=keyVault_URI, credential=credential) data_access_key = client.get_secret("thecupstore-key") account_url = "https://thecupstore.queue.core.windows.net/" queueclient = QueueClient(account_url=account_url, queue_name=queue_name, credential=data_access_key.value, message_encode_policy=TextBase64EncodePolicy(), message_decode_policy=TextBase64DecodePolicy()) # Check that the queue exists and if not create it if the create switch has been passed as True try: queueclient.get_queue_properties() except: if create_queue: queueclient.create_queue() else: message = "Queue does not exist" else: if clear_queue: queueclient.clear_messages() if 'message' in locals(): # checks for existence of message variable return message else: return queueclient
def test_message_base64_decode_fails(self, resource_group, location, storage_account, storage_account_key): # Arrange qsc = QueueServiceClient(self._account_url(storage_account.name), storage_account_key) queue = QueueClient( account_url=self._account_url(storage_account.name), queue_name=self.get_resource_name(TEST_QUEUE_PREFIX), credential=storage_account_key, message_encode_policy=None, message_decode_policy=BinaryBase64DecodePolicy()) try: queue.create_queue() except ResourceExistsError: pass message = u'xyz' queue.send_message(message) # Action. with self.assertRaises(DecodeError) as e: queue.peek_messages() # Asserts self.assertNotEqual( -1, str(e.exception).find('Message content is not valid base 64'))
def queue_and_messages_example(self): # Instantiate the QueueClient from a connection string from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string( conn_str=self.connection_string, queue_name="myqueue") # Create the queue # [START create_queue] queue.create_queue() # [END create_queue] try: # Send messages queue.send_message(u"I'm using queues!") queue.send_message(u"This is my second message") # Receive the messages response = queue.receive_messages(messages_per_page=2) # Print the content of the messages for message in response: print(message.content) finally: # [START delete_queue] queue.delete_queue()
def test_delete_and_clear_messages(self, resource_group, location, storage_account, storage_account_key): # Instantiate a queue client from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string(self.connection_string(storage_account, storage_account_key), "delqueue") # Create the queue queue.create_queue() try: # Send messages queue.send_message(u"message1") queue.send_message(u"message2") queue.send_message(u"message3") queue.send_message(u"message4") queue.send_message(u"message5") # [START delete_message] # Get the message at the front of the queue msg = next(queue.receive_messages()) # Delete the specified message queue.delete_message(msg) # [END delete_message] # [START clear_messages] queue.clear_messages() # [END clear_messages] finally: # Delete the queue queue.delete_queue()
def get_projects_via_queue(): print("Projects queue is live") mycol = refer_projects_col() connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING') projects_client = QueueClient.from_connection_string(connect_str, "projects-queue") while (True): time.sleep(10) rows = projects_client.receive_messages() # print(rows) for msg in rows: # time.sleep(60) row = msg.content row = ast.literal_eval(row) print(row[0],' processing queries from the key phrases') entry_id = ObjectId(row[0]) proj_data_entry = mycol.find({"_id": entry_id}) data = [i for i in proj_data_entry] # print(data[0]) key_phrases = data[0]['key_phrases'] queries = process_queries(key_phrases) query_count = len(queries) mycol.update_one({'_id': entry_id }, {'$set': {'query_count': query_count}}) for each_query in queries: print(each_query," adding to pipeline execution") add_to_initial_crawling_queue([each_query+' ++'+str(entry_id)+' --project']) projects_client.delete_message(msg) add_to_project_completion_queue([entry_id])
def test_sas_read(self, resource_group, location, storage_account, storage_account_key): # SAS URL is calculated from storage key, so this test runs live only # Arrange qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key) queue_client = self._get_queue_reference(qsc) queue_client.create_queue() queue_client.send_message(u'message1') token = generate_queue_sas( queue_client.account_name, queue_client.queue_name, queue_client.credential.account_key, QueueSasPermissions(read=True), datetime.utcnow() + timedelta(hours=1), datetime.utcnow() - timedelta(minutes=5) ) # Act service = QueueClient.from_queue_url( queue_url=queue_client.url, credential=token, ) result = service.peek_messages() # Assert self.assertIsNotNone(result) self.assertEqual(1, len(result)) message = result[0] self.assertIsNotNone(message) self.assertNotEqual('', message.id) self.assertEqual(u'message1', message.content)
def add_to_project_completion_queue(id_list): connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING') projects_client = QueueClient.from_connection_string(connect_str, "project-completion-queue") for each_id in id_list: print(each_id," added to projects completion queue") projects_client.send_message([str(each_id)], time_to_live=-1)
def main(msg: func.QueueMessage) -> None: queue_client = QueueClient.from_connection_string( os.environ['AzureWebJobsStorage'], 'oat-queue', message_encode_policy=TextBase64EncodePolicy(), ) queue_client.send_message(msg.get_body().decode(), visibility_timeout=3600)
def receive_one_message_from_queue(self): # Instantiate a queue client from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string(self.connection_string, "myqueue5") # Create the queue queue.create_queue() try: queue.send_message(u"message1") queue.send_message(u"message2") queue.send_message(u"message3") # [START receive_one_message] # Pop two messages from the front of the queue message1 = queue.receive_message() message2 = queue.receive_message() # We should see message 3 if we peek message3 = queue.peek_messages()[0] print(message1.content) print(message2.content) print(message3.content) # [END receive_one_message] finally: queue.delete_queue()
def test_peek_messages(self, resource_group, location, storage_account, storage_account_key): # Instantiate a queue client from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string(self.connection_string(storage_account, storage_account_key), "peekqueue") # Create the queue queue.create_queue() try: # Send messages queue.send_message(u"message1") queue.send_message(u"message2") queue.send_message(u"message3") queue.send_message(u"message4") queue.send_message(u"message5") # [START peek_message] # Peek at one message at the front of the queue msg = queue.peek_messages() # Peek at the last 5 messages messages = queue.peek_messages(max_messages=5) # Print the last 5 messages for message in messages: print(message.content) # [END peek_message] finally: # Delete the queue queue.delete_queue()
def peek_messages(self): # Instantiate a queue client from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string(self.connection_string, "myqueue5") # Create the queue queue.create_queue() try: # Send messages queue.send_message(u"message1") queue.send_message(u"message2") queue.send_message(u"message3") queue.send_message(u"message4") queue.send_message(u"message5") # [START peek_message] # Peek at one message at the front of the queue msg = queue.peek_messages() # Peek at the last 5 messages messages = queue.peek_messages(max_messages=5) # Print the last 5 messages for message in messages: print(message.content) # [END peek_message] finally: # Delete the queue queue.delete_queue()
def test_sas_add(self, resource_group, location, storage_account, storage_account_key): # SAS URL is calculated from storage key, so this test runs live only # Arrange qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key) queue_client = self._get_queue_reference(qsc) queue_client.create_queue() token = generate_queue_sas( queue_client.account_name, queue_client.queue_name, queue_client.credential.account_key, QueueSasPermissions(add=True), datetime.utcnow() + timedelta(hours=1), ) # Act service = QueueClient.from_queue_url( queue_url=queue_client.url, credential=token, ) result = service.send_message(u'addedmessage') # Assert result = next(queue_client.receive_messages()) self.assertEqual(u'addedmessage', result.content)
def add_to_initial_crawling_queue(name_list): mycol = refer_collection() connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING') ic_client = QueueClient.from_connection_string(connect_str, "initial-crawling-queue") for name in name_list: print(name) ic_client.send_message([str(name)])
def delete_and_clear_messages(self): # Instantiate a queue client from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string(self.connection_string, "myqueue4") # Create the queue queue.create_queue() try: # Send messages queue.send_message(u"message1") queue.send_message(u"message2") queue.send_message(u"message3") queue.send_message(u"message4") queue.send_message(u"message5") # [START delete_message] # Get the message at the front of the queue msg = next(queue.receive_messages()) # Delete the specified message queue.delete_message(msg) # [END delete_message] # [START clear_messages] queue.clear_messages() # [END clear_messages] finally: # Delete the queue queue.delete_queue()
def test_update_message(self, resource_group, location, storage_account, storage_account_key): # Instantiate a queue client from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string(self.connection_string(storage_account, storage_account_key), "updatequeue") # Create the queue queue.create_queue() try: # [START update_message] # Send a message queue.send_message(u"update me") # Receive the message messages = queue.receive_messages() # Update the message list_result = next(messages) message = queue.update_message( list_result.id, pop_receipt=list_result.pop_receipt, visibility_timeout=0, content=u"updated") # [END update_message] assert message.content == "updated" finally: # Delete the queue queue.delete_queue()
def update_message(self): # Instantiate a queue client from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string(self.connection_string, "myqueue6") # Create the queue queue.create_queue() try: # [START update_message] # Send a message queue.send_message(u"update me") # Receive the message messages = queue.receive_messages() # Update the message list_result = next(messages) message = queue.update_message(list_result.id, pop_receipt=list_result.pop_receipt, visibility_timeout=0, content=u"updated") # [END update_message] finally: # Delete the queue queue.delete_queue()
def list_message_pages(self): # Instantiate a queue client from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string(self.connection_string, "myqueue4") # Create the queue queue.create_queue() try: queue.send_message(u"message1") queue.send_message(u"message2") queue.send_message(u"message3") queue.send_message(u"message4") queue.send_message(u"message5") queue.send_message(u"message6") # [START receive_messages_listing] # Store two messages in each page message_batches = queue.receive_messages( messages_per_page=2).by_page() # Iterate through the page lists print(list(next(message_batches))) print(list(next(message_batches))) # There are two iterations in the last page as well. last_page = next(message_batches) for message in last_page: print(message) # [END receive_messages_listing] finally: queue.delete_queue()