def run(self):
'''Runs the pipeline step.
'''
queue = QueueClient(account_url=os.getenv('AZ_QS_AC_URL'),
queue_name=os.getenv('AZ_QS_QUEUE_NAME'),
credential=os.getenv('AZ_QS_SAS_TOKEN'))
response = queue.receive_messages(messages_per_page=5)
for batch in response.by_page():
for message in batch:
filename = message.content
label, _ = classifier.predict(filename)
self.print('\'{filename}\' classified as \'{label}\'',
filename=filename,
label=label['name'])
with open(self.output['classifier'], 'a') as f:
f.write(message.content + ',' + str(label['id']) + '\n')
if label['name'] != 'other':
entities = ner.predict(filename)
self.print('\'{filename}\' has entities \'{entities}\'',
filename=filename,
entities=entities)
with open(self.output['ner'], 'a') as f:
f.write(message.content + ',\"' + str(entities) +
'\"\n')
queue.delete_message(message)
def test_sas_process(self, resource_group, location, storage_account,
storage_account_key):
# SAS URL is calculated from storage key, so this test runs live only
if not self.is_live:
return
# Arrange
qsc = QueueServiceClient(self._account_url(storage_account.name),
storage_account_key)
queue_client = self._get_queue_reference(qsc)
queue_client.create_queue()
queue_client.enqueue_message(u'message1')
token = queue_client.generate_shared_access_signature(
QueueSasPermissions(process=True),
datetime.utcnow() + timedelta(hours=1),
)
# Act
service = QueueClient(
queue_url=queue_client.url,
credential=token,
)
message = next(service.receive_messages())
# Assert
self.assertIsNotNone(message)
self.assertNotEqual('', message.id)
self.assertEqual(u'message1', message.content)
def test_set_access_policy(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# [START create_queue_client_from_connection_string]
from azure.storage.queue import QueueClient
queue_client = QueueClient.from_connection_string(self.connection_string, "queuetest")
# [END create_queue_client_from_connection_string]
# Create the queue
try:
queue_client.create_queue()
except ResourceExistsError:
pass
queue_client.enqueue_message(u"hello world")
try:
# [START set_access_policy]
# Create an access policy
from azure.storage.queue import AccessPolicy, QueuePermissions
access_policy = AccessPolicy()
access_policy.start = datetime.utcnow() - timedelta(hours=1)
access_policy.expiry = datetime.utcnow() + timedelta(hours=1)
access_policy.permission = QueuePermissions.READ
identifiers = {'my-access-policy-id': access_policy}
# Set the access policy
queue_client.set_queue_access_policy(identifiers)
# [END set_access_policy]
# Use the access policy to generate a SAS token
# [START queue_client_sas_token]
sas_token = queue_client.generate_shared_access_signature(
policy_id='my-access-policy-id'
)
# [END queue_client_sas_token]
# Authenticate with the sas token
# [START create_queue_client]
q = QueueClient(
queue_url=queue_client.url,
credential=sas_token
)
# [END create_queue_client]
# Use the newly authenticated client to receive messages
my_message = q.receive_messages()
assert my_message is not None
finally:
# Delete the queue
queue_client.delete_queue()
def main(accountName, queueName):
accountURL = "https://%s.queue.core.windows.net" % (accountName)
creds = ManagedIdentityCredential()
client = QueueClient(account_url=accountURL,
queue_name=queueName,
credential=creds)
messages = client.receive_messages(messages_per_page=1)
for message in messages:
print(message.content)
client.delete_message(message)
def _pop_specific_q(_q: QueueClient, _n: int) -> bool:
has_messages = False
for m in _q.receive_messages(messages_per_page=_n):
if m:
has_messages = True
result.append(m if raw else self._deserialize_message(m))
if delete:
_q.delete_message(m.id, m.pop_receipt)
# short circuit to prevent unneeded work
if len(result) == n:
return True
return has_messages
def test_sas_process(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
queue_client = self._create_queue()
queue_client.enqueue_message(u'message1')
token = queue_client.generate_shared_access_signature(
QueuePermissions.PROCESS,
datetime.utcnow() + timedelta(hours=1),
)
# Act
service = QueueClient(
queue_url=queue_client.url,
credential=token,
)
message = next(service.receive_messages())
# Assert
self.assertIsNotNone(message)
self.assertNotEqual('', message.id)
self.assertEqual(u'message1', message.content)
str(metadata.approximate_message_count))
time.sleep(1)
###
# Use the Azure Storage Storage SDK for Python to read each message from the Queue
###
print(
'\nWith some messages in our Azure Storage Queue, let\'s read the first message in the Queue to signal we start to process that customer\'s order.'
)
raw_input('Press Enter to continue...')
# When you get each message, they become hidden from other parts of the applications being able to see it.
# Once you have successfully processed the message, you then delete the message from the Queue.
# This behavior makes sure that if something goes wrong in the processing of the message, it is then dropped back in the Queue for processing in the next cycle.
messages = queue_client.receive_messages()
for message in messages:
print('\n' + message.content)
queue_client.delete_message(message.id, message.pop_receipt)
raw_input('\nPress Enter to continue...')
metadata = queue_client.get_queue_properties()
print(
'If we look at the Queue again, we have one less message to show we have processed that order and a yummy pizza will be on it\'s way to the customer soon.'
)
print('Number of messages in the queue: ' +
str(metadata.approximate_message_count))
raw_input('\nPress Enter to continue...')
###