def test_sas_add(self, resource_group, location, storage_account,
storage_account_key):
# SAS URL is calculated from storage key, so this test runs live only
if not self.is_live:
return
# Arrange
qsc = QueueServiceClient(self._account_url(storage_account.name),
storage_account_key)
queue_client = self._get_queue_reference(qsc)
queue_client.create_queue()
token = queue_client.generate_shared_access_signature(
QueuePermissions.ADD,
datetime.utcnow() + timedelta(hours=1),
)
# Act
service = QueueClient(
queue_url=queue_client.url,
credential=token,
)
result = service.enqueue_message(u'addedmessage')
# Assert
result = next(queue_client.receive_messages())
self.assertEqual(u'addedmessage', result.content)
def test_sas_process(self, resource_group, location, storage_account,
storage_account_key):
# SAS URL is calculated from storage key, so this test runs live only
if not self.is_live:
return
# Arrange
qsc = QueueServiceClient(self._account_url(storage_account.name),
storage_account_key)
queue_client = self._get_queue_reference(qsc)
queue_client.create_queue()
queue_client.enqueue_message(u'message1')
token = queue_client.generate_shared_access_signature(
QueueSasPermissions(process=True),
datetime.utcnow() + timedelta(hours=1),
)
# Act
service = QueueClient(
queue_url=queue_client.url,
credential=token,
)
message = next(service.receive_messages())
# Assert
self.assertIsNotNone(message)
self.assertNotEqual('', message.id)
self.assertEqual(u'message1', message.content)
def test_sas_read(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
queue_client = self._create_queue()
queue_client.enqueue_message(u'message1')
token = queue_client.generate_shared_access_signature(
QueuePermissions.READ,
datetime.utcnow() + timedelta(hours=1),
datetime.utcnow() - timedelta(minutes=5))
# Act
service = QueueClient(
queue_url=queue_client.url,
credential=token,
)
result = service.peek_messages()
# Assert
self.assertIsNotNone(result)
self.assertEqual(1, len(result))
message = result[0]
self.assertIsNotNone(message)
self.assertNotEqual('', message.id)
self.assertEqual(u'message1', message.content)
def test_sas_update(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
queue_client = self._create_queue()
queue_client.enqueue_message(u'message1')
token = queue_client.generate_shared_access_signature(
QueuePermissions.UPDATE,
datetime.utcnow() + timedelta(hours=1),
)
messages = queue_client.receive_messages()
result = next(messages)
# Act
service = QueueClient(
queue_url=queue_client.url,
credential=token,
)
service.update_message(
result.id,
pop_receipt=result.pop_receipt,
visibility_timeout=0,
content=u'updatedmessage1',
)
# Assert
result = next(messages)
self.assertEqual(u'updatedmessage1', result.content)
def test_sas_update(self, resource_group, location, storage_account,
storage_account_key):
# SAS URL is calculated from storage key, so this test runs live only
if not self.is_live:
return
# Arrange
qsc = QueueServiceClient(self._account_url(storage_account.name),
storage_account_key)
queue_client = self._get_queue_reference(qsc)
queue_client.create_queue()
queue_client.enqueue_message(u'message1')
token = queue_client.generate_shared_access_signature(
QueueSasPermissions(update=True),
datetime.utcnow() + timedelta(hours=1),
)
messages = queue_client.receive_messages()
result = next(messages)
# Act
service = QueueClient(
queue_url=queue_client.url,
credential=token,
)
service.update_message(
result.id,
pop_receipt=result.pop_receipt,
visibility_timeout=0,
content=u'updatedmessage1',
)
# Assert
result = next(messages)
self.assertEqual(u'updatedmessage1', result.content)
def test_sas_signed_identifier(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
access_policy = AccessPolicy()
access_policy.start = datetime.utcnow() - timedelta(hours=1)
access_policy.expiry = datetime.utcnow() + timedelta(hours=1)
access_policy.permission = QueuePermissions.READ
identifiers = {'testid': access_policy}
queue_client = self._create_queue()
resp = queue_client.set_queue_access_policy(identifiers)
queue_client.enqueue_message(u'message1')
token = queue_client.generate_shared_access_signature(
policy_id='testid')
# Act
service = QueueClient(
queue_url=queue_client.url,
credential=token,
)
result = service.peek_messages()
# Assert
self.assertIsNotNone(result)
self.assertEqual(1, len(result))
message = result[0]
self.assertIsNotNone(message)
self.assertNotEqual('', message.id)
self.assertEqual(u'message1', message.content)
def test_create_service_with_custom_account_endpoint_path(self, resource_group, location, storage_account, storage_account_key):
custom_account_url = "http://local-machine:11002/custom/account/path/" + self.sas_token
for service_type in SERVICES.items():
conn_string = 'DefaultEndpointsProtocol=http;AccountName={};AccountKey={};QueueEndpoint={};'.format(
storage_account.name, storage_account_key, custom_account_url)
# Act
service = service_type[0].from_connection_string(conn_string, queue_name="foo")
# Assert
self.assertEqual(service.account_name, storage_account.name)
self.assertEqual(service.credential.account_name, storage_account.name)
self.assertEqual(service.credential.account_key, storage_account_key)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
service = QueueServiceClient(account_url=custom_account_url)
self.assertEqual(service.account_name, None)
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/?'))
service = QueueClient(account_url=custom_account_url, queue_name="foo")
self.assertEqual(service.account_name, None)
self.assertEqual(service.queue_name, "foo")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/foo?'))
service = QueueClient.from_queue_url("http://local-machine:11002/custom/account/path/foo" + self.sas_token)
self.assertEqual(service.account_name, None)
self.assertEqual(service.queue_name, "foo")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/foo?'))
def run(self):
'''Runs the pipeline step.
'''
queue = QueueClient(account_url=os.getenv('AZ_QS_AC_URL'),
queue_name=os.getenv('AZ_QS_QUEUE_NAME'),
credential=os.getenv('AZ_QS_SAS_TOKEN'))
response = queue.receive_messages(messages_per_page=5)
for batch in response.by_page():
for message in batch:
filename = message.content
label, _ = classifier.predict(filename)
self.print('\'{filename}\' classified as \'{label}\'',
filename=filename,
label=label['name'])
with open(self.output['classifier'], 'a') as f:
f.write(message.content + ',' + str(label['id']) + '\n')
if label['name'] != 'other':
entities = ner.predict(filename)
self.print('\'{filename}\' has entities \'{entities}\'',
filename=filename,
entities=entities)
with open(self.output['ner'], 'a') as f:
f.write(message.content + ',\"' + str(entities) +
'\"\n')
queue.delete_message(message)
def test_set_access_policy(self, resource_group, location, storage_account, storage_account_key):
# SAS URL is calculated from storage key, so this test runs live only
if not self.is_live:
return
# [START create_queue_client_from_connection_string]
from azure.storage.queue import QueueClient
queue_client = QueueClient.from_connection_string(self.connection_string(storage_account, storage_account_key), "queuetest")
# [END create_queue_client_from_connection_string]
# Create the queue
try:
queue_client.create_queue()
except ResourceExistsError:
pass
queue_client.send_message(u"hello world")
try:
# [START set_access_policy]
# Create an access policy
from azure.storage.queue import AccessPolicy, QueueSasPermissions
access_policy = AccessPolicy()
access_policy.start = datetime.utcnow() - timedelta(hours=1)
access_policy.expiry = datetime.utcnow() + timedelta(hours=1)
access_policy.permission = QueueSasPermissions(read=True)
identifiers = {'my-access-policy-id': access_policy}
# Set the access policy
queue_client.set_queue_access_policy(identifiers)
# [END set_access_policy]
# Use the access policy to generate a SAS token
# [START queue_client_sas_token]
from azure.storage.queue import generate_queue_sas
sas_token = generate_queue_sas(
queue_client.account_name,
queue_client.queue_name,
queue_client.credential.account_key,
policy_id='my-access-policy-id'
)
# [END queue_client_sas_token]
# Authenticate with the sas token
# [START create_queue_client]
q = QueueClient.from_queue_url(
queue_url=queue_client.url,
credential=sas_token
)
# [END create_queue_client]
# Use the newly authenticated client to receive messages
my_message = q.receive_messages()
assert my_message is not None
finally:
# Delete the queue
queue_client.delete_queue()
def test_set_access_policy(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# [START create_queue_client_from_connection_string]
from azure.storage.queue import QueueClient
queue_client = QueueClient.from_connection_string(self.connection_string, "queuetest")
# [END create_queue_client_from_connection_string]
# Create the queue
try:
queue_client.create_queue()
except ResourceExistsError:
pass
queue_client.enqueue_message(u"hello world")
try:
# [START set_access_policy]
# Create an access policy
from azure.storage.queue import AccessPolicy, QueuePermissions
access_policy = AccessPolicy()
access_policy.start = datetime.utcnow() - timedelta(hours=1)
access_policy.expiry = datetime.utcnow() + timedelta(hours=1)
access_policy.permission = QueuePermissions.READ
identifiers = {'my-access-policy-id': access_policy}
# Set the access policy
queue_client.set_queue_access_policy(identifiers)
# [END set_access_policy]
# Use the access policy to generate a SAS token
# [START queue_client_sas_token]
sas_token = queue_client.generate_shared_access_signature(
policy_id='my-access-policy-id'
)
# [END queue_client_sas_token]
# Authenticate with the sas token
# [START create_queue_client]
q = QueueClient(
queue_url=queue_client.url,
credential=sas_token
)
# [END create_queue_client]
# Use the newly authenticated client to receive messages
my_message = q.receive_messages()
assert my_message is not None
finally:
# Delete the queue
queue_client.delete_queue()
def main(accountName, queueName):
accountURL = "https://%s.queue.core.windows.net" % (accountName)
creds = ManagedIdentityCredential()
client = QueueClient(account_url=accountURL,
queue_name=queueName,
credential=creds)
messages = client.receive_messages(messages_per_page=1)
for message in messages:
print(message.content)
client.delete_message(message)
def _pop_specific_q(_q: QueueClient, _n: int) -> bool:
has_messages = False
for m in _q.receive_messages(messages_per_page=_n):
if m:
has_messages = True
result.append(m if raw else self._deserialize_message(m))
if delete:
_q.delete_message(m.id, m.pop_receipt)
# short circuit to prevent unneeded work
if len(result) == n:
return True
return has_messages
def set_access_policy(self):
# [START create_queue_client_from_connection_string]
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(self.connection_string, "myqueue1")
# [END create_queue_client_from_connection_string]
# Create the queue
queue.create_queue()
# Send a message
queue.send_message(u"hello world")
try:
# [START set_access_policy]
# Create an access policy
from azure.storage.queue import AccessPolicy, QueueSasPermissions
access_policy = AccessPolicy()
access_policy.start = datetime.utcnow() - timedelta(hours=1)
access_policy.expiry = datetime.utcnow() + timedelta(hours=1)
access_policy.permission = QueueSasPermissions(read=True)
identifiers = {'my-access-policy-id': access_policy}
# Set the access policy
queue.set_queue_access_policy(identifiers)
# [END set_access_policy]
# Use the access policy to generate a SAS token
# [START queue_client_sas_token]
from azure.storage.queue import generate_queue_sas
sas_token = generate_queue_sas(
queue.account_name,
queue.queue_name,
queue.credential.account_key,
policy_id='my-access-policy-id'
)
# [END queue_client_sas_token]
# Authenticate with the sas token
# [START create_queue_client]
token_auth_queue = QueueClient.from_queue_url(
queue_url=queue.url,
credential=sas_token
)
# [END create_queue_client]
# Use the newly authenticated client to receive messages
my_message = token_auth_queue.receive_messages()
finally:
# Delete the queue
queue.delete_queue()
def get_queue(queue_name, create_queue, clear_queue):
''' Note that generating the queueclient does not mean there must a queue there as one of the properties of queueclient is "create_queue", so it's
really a representation of a queue which may or may not exist yet. '''
keyVaultName = os.environ["KEY_VAULT_NAME"]
keyVault_URI = "https://" + keyVaultName + ".vault.azure.net"
credential = DefaultAzureCredential()
client = SecretClient(vault_url=keyVault_URI, credential=credential)
data_access_key = client.get_secret("thecupstore-key")
account_url = "https://thecupstore.queue.core.windows.net/"
queueclient = QueueClient(account_url=account_url,
queue_name=queue_name,
credential=data_access_key.value,
message_encode_policy=TextBase64EncodePolicy(),
message_decode_policy=TextBase64DecodePolicy())
# Check that the queue exists and if not create it if the create switch has been passed as True
try:
queueclient.get_queue_properties()
except:
if create_queue:
queueclient.create_queue()
else:
message = "Queue does not exist"
else:
if clear_queue:
queueclient.clear_messages()
if 'message' in locals(): # checks for existence of message variable
return message
else:
return queueclient
def test_message_base64_decode_fails(self, resource_group, location,
storage_account, storage_account_key):
# Arrange
qsc = QueueServiceClient(self._account_url(storage_account.name),
storage_account_key)
queue = QueueClient(
account_url=self._account_url(storage_account.name),
queue_name=self.get_resource_name(TEST_QUEUE_PREFIX),
credential=storage_account_key,
message_encode_policy=None,
message_decode_policy=BinaryBase64DecodePolicy())
try:
queue.create_queue()
except ResourceExistsError:
pass
message = u'xyz'
queue.send_message(message)
# Action.
with self.assertRaises(DecodeError) as e:
queue.peek_messages()
# Asserts
self.assertNotEqual(
-1,
str(e.exception).find('Message content is not valid base 64'))
def queue_and_messages_example(self):
# Instantiate the QueueClient from a connection string
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(
conn_str=self.connection_string, queue_name="myqueue")
# Create the queue
# [START create_queue]
queue.create_queue()
# [END create_queue]
try:
# Send messages
queue.send_message(u"I'm using queues!")
queue.send_message(u"This is my second message")
# Receive the messages
response = queue.receive_messages(messages_per_page=2)
# Print the content of the messages
for message in response:
print(message.content)
finally:
# [START delete_queue]
queue.delete_queue()
def test_delete_and_clear_messages(self, resource_group, location, storage_account, storage_account_key):
# Instantiate a queue client
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(self.connection_string(storage_account, storage_account_key), "delqueue")
# Create the queue
queue.create_queue()
try:
# Send messages
queue.send_message(u"message1")
queue.send_message(u"message2")
queue.send_message(u"message3")
queue.send_message(u"message4")
queue.send_message(u"message5")
# [START delete_message]
# Get the message at the front of the queue
msg = next(queue.receive_messages())
# Delete the specified message
queue.delete_message(msg)
# [END delete_message]
# [START clear_messages]
queue.clear_messages()
# [END clear_messages]
finally:
# Delete the queue
queue.delete_queue()
def get_projects_via_queue():
print("Projects queue is live")
mycol = refer_projects_col()
connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING')
projects_client = QueueClient.from_connection_string(connect_str, "projects-queue")
while (True):
time.sleep(10)
rows = projects_client.receive_messages()
# print(rows)
for msg in rows:
# time.sleep(60)
row = msg.content
row = ast.literal_eval(row)
print(row[0],' processing queries from the key phrases')
entry_id = ObjectId(row[0])
proj_data_entry = mycol.find({"_id": entry_id})
data = [i for i in proj_data_entry]
# print(data[0])
key_phrases = data[0]['key_phrases']
queries = process_queries(key_phrases)
query_count = len(queries)
mycol.update_one({'_id': entry_id },
{'$set': {'query_count': query_count}})
for each_query in queries:
print(each_query," adding to pipeline execution")
add_to_initial_crawling_queue([each_query+' ++'+str(entry_id)+' --project'])
projects_client.delete_message(msg)
add_to_project_completion_queue([entry_id])
def test_sas_read(self, resource_group, location, storage_account, storage_account_key):
# SAS URL is calculated from storage key, so this test runs live only
# Arrange
qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key)
queue_client = self._get_queue_reference(qsc)
queue_client.create_queue()
queue_client.send_message(u'message1')
token = generate_queue_sas(
queue_client.account_name,
queue_client.queue_name,
queue_client.credential.account_key,
QueueSasPermissions(read=True),
datetime.utcnow() + timedelta(hours=1),
datetime.utcnow() - timedelta(minutes=5)
)
# Act
service = QueueClient.from_queue_url(
queue_url=queue_client.url,
credential=token,
)
result = service.peek_messages()
# Assert
self.assertIsNotNone(result)
self.assertEqual(1, len(result))
message = result[0]
self.assertIsNotNone(message)
self.assertNotEqual('', message.id)
self.assertEqual(u'message1', message.content)
def add_to_project_completion_queue(id_list):
connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING')
projects_client = QueueClient.from_connection_string(connect_str, "project-completion-queue")
for each_id in id_list:
print(each_id," added to projects completion queue")
projects_client.send_message([str(each_id)], time_to_live=-1)
def main(msg: func.QueueMessage) -> None:
queue_client = QueueClient.from_connection_string(
os.environ['AzureWebJobsStorage'],
'oat-queue',
message_encode_policy=TextBase64EncodePolicy(),
)
queue_client.send_message(msg.get_body().decode(), visibility_timeout=3600)
def receive_one_message_from_queue(self):
# Instantiate a queue client
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(self.connection_string,
"myqueue5")
# Create the queue
queue.create_queue()
try:
queue.send_message(u"message1")
queue.send_message(u"message2")
queue.send_message(u"message3")
# [START receive_one_message]
# Pop two messages from the front of the queue
message1 = queue.receive_message()
message2 = queue.receive_message()
# We should see message 3 if we peek
message3 = queue.peek_messages()[0]
print(message1.content)
print(message2.content)
print(message3.content)
# [END receive_one_message]
finally:
queue.delete_queue()
def test_peek_messages(self, resource_group, location, storage_account, storage_account_key):
# Instantiate a queue client
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(self.connection_string(storage_account, storage_account_key), "peekqueue")
# Create the queue
queue.create_queue()
try:
# Send messages
queue.send_message(u"message1")
queue.send_message(u"message2")
queue.send_message(u"message3")
queue.send_message(u"message4")
queue.send_message(u"message5")
# [START peek_message]
# Peek at one message at the front of the queue
msg = queue.peek_messages()
# Peek at the last 5 messages
messages = queue.peek_messages(max_messages=5)
# Print the last 5 messages
for message in messages:
print(message.content)
# [END peek_message]
finally:
# Delete the queue
queue.delete_queue()
def peek_messages(self):
# Instantiate a queue client
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(self.connection_string,
"myqueue5")
# Create the queue
queue.create_queue()
try:
# Send messages
queue.send_message(u"message1")
queue.send_message(u"message2")
queue.send_message(u"message3")
queue.send_message(u"message4")
queue.send_message(u"message5")
# [START peek_message]
# Peek at one message at the front of the queue
msg = queue.peek_messages()
# Peek at the last 5 messages
messages = queue.peek_messages(max_messages=5)
# Print the last 5 messages
for message in messages:
print(message.content)
# [END peek_message]
finally:
# Delete the queue
queue.delete_queue()
def test_sas_add(self, resource_group, location, storage_account, storage_account_key):
# SAS URL is calculated from storage key, so this test runs live only
# Arrange
qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key)
queue_client = self._get_queue_reference(qsc)
queue_client.create_queue()
token = generate_queue_sas(
queue_client.account_name,
queue_client.queue_name,
queue_client.credential.account_key,
QueueSasPermissions(add=True),
datetime.utcnow() + timedelta(hours=1),
)
# Act
service = QueueClient.from_queue_url(
queue_url=queue_client.url,
credential=token,
)
result = service.send_message(u'addedmessage')
# Assert
result = next(queue_client.receive_messages())
self.assertEqual(u'addedmessage', result.content)
def add_to_initial_crawling_queue(name_list):
mycol = refer_collection()
connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING')
ic_client = QueueClient.from_connection_string(connect_str, "initial-crawling-queue")
for name in name_list:
print(name)
ic_client.send_message([str(name)])
def delete_and_clear_messages(self):
# Instantiate a queue client
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(self.connection_string,
"myqueue4")
# Create the queue
queue.create_queue()
try:
# Send messages
queue.send_message(u"message1")
queue.send_message(u"message2")
queue.send_message(u"message3")
queue.send_message(u"message4")
queue.send_message(u"message5")
# [START delete_message]
# Get the message at the front of the queue
msg = next(queue.receive_messages())
# Delete the specified message
queue.delete_message(msg)
# [END delete_message]
# [START clear_messages]
queue.clear_messages()
# [END clear_messages]
finally:
# Delete the queue
queue.delete_queue()
def test_update_message(self, resource_group, location, storage_account, storage_account_key):
# Instantiate a queue client
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(self.connection_string(storage_account, storage_account_key), "updatequeue")
# Create the queue
queue.create_queue()
try:
# [START update_message]
# Send a message
queue.send_message(u"update me")
# Receive the message
messages = queue.receive_messages()
# Update the message
list_result = next(messages)
message = queue.update_message(
list_result.id,
pop_receipt=list_result.pop_receipt,
visibility_timeout=0,
content=u"updated")
# [END update_message]
assert message.content == "updated"
finally:
# Delete the queue
queue.delete_queue()
def update_message(self):
# Instantiate a queue client
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(self.connection_string,
"myqueue6")
# Create the queue
queue.create_queue()
try:
# [START update_message]
# Send a message
queue.send_message(u"update me")
# Receive the message
messages = queue.receive_messages()
# Update the message
list_result = next(messages)
message = queue.update_message(list_result.id,
pop_receipt=list_result.pop_receipt,
visibility_timeout=0,
content=u"updated")
# [END update_message]
finally:
# Delete the queue
queue.delete_queue()
def list_message_pages(self):
# Instantiate a queue client
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(self.connection_string,
"myqueue4")
# Create the queue
queue.create_queue()
try:
queue.send_message(u"message1")
queue.send_message(u"message2")
queue.send_message(u"message3")
queue.send_message(u"message4")
queue.send_message(u"message5")
queue.send_message(u"message6")
# [START receive_messages_listing]
# Store two messages in each page
message_batches = queue.receive_messages(
messages_per_page=2).by_page()
# Iterate through the page lists
print(list(next(message_batches)))
print(list(next(message_batches)))
# There are two iterations in the last page as well.
last_page = next(message_batches)
for message in last_page:
print(message)
# [END receive_messages_listing]
finally:
queue.delete_queue()