def test_create_service_with_custom_account_endpoint_path(self, resource_group, location, storage_account, storage_account_key):
custom_account_url = "http://local-machine:11002/custom/account/path/" + self.sas_token
for service_type in SERVICES.items():
conn_string = 'DefaultEndpointsProtocol=http;AccountName={};AccountKey={};QueueEndpoint={};'.format(
storage_account.name, storage_account_key, custom_account_url)
# Act
service = service_type[0].from_connection_string(conn_string, queue_name="foo")
# Assert
self.assertEqual(service.account_name, storage_account.name)
self.assertEqual(service.credential.account_name, storage_account.name)
self.assertEqual(service.credential.account_key, storage_account_key)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
service = QueueServiceClient(account_url=custom_account_url)
self.assertEqual(service.account_name, None)
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/?'))
service = QueueClient(account_url=custom_account_url, queue_name="foo")
self.assertEqual(service.account_name, None)
self.assertEqual(service.queue_name, "foo")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/foo?'))
service = QueueClient.from_queue_url("http://local-machine:11002/custom/account/path/foo" + self.sas_token)
self.assertEqual(service.account_name, None)
self.assertEqual(service.queue_name, "foo")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/foo?'))
async def test_set_access_policy(self, resource_group, location,
storage_account, storage_account_key):
connection_string = self.connection_string(storage_account,
storage_account_key)
storage_url = self._account_url(storage_account.name)
# [START async_create_queue_client_from_connection_string]
from azure.storage.queue.aio import QueueClient
queue_client = QueueClient.from_connection_string(
self.connection_string(storage_account, storage_account_key),
"asyncqueuetest")
# [END async_create_queue_client_from_connection_string]
# Create the queue
try:
await queue_client.create_queue()
except ResourceExistsError:
pass
await queue_client.send_message(u"hello world")
try:
# [START async_set_access_policy]
# Create an access policy
from azure.storage.queue import AccessPolicy, QueueSasPermissions
access_policy = AccessPolicy()
access_policy.start = datetime.utcnow() - timedelta(hours=1)
access_policy.expiry = datetime.utcnow() + timedelta(hours=1)
access_policy.permission = QueueSasPermissions(read=True)
identifiers = {'my-access-policy-id': access_policy}
# Set the access policy
await queue_client.set_queue_access_policy(identifiers)
# [END async_set_access_policy]
# Use the access policy to generate a SAS token
from azure.storage.queue import generate_queue_sas
sas_token = generate_queue_sas(queue_client.account_name,
queue_client.queue_name,
queue_client.credential.account_key,
policy_id='my-access-policy-id')
# [END async_set_access_policy]
# Authenticate with the sas token
# [START async_create_queue_client]
from azure.storage.queue.aio import QueueClient
q = QueueClient.from_queue_url(queue_url=queue_client.url,
credential=sas_token)
# [END async_create_queue_client]
# Use the newly authenticated client to receive messages
my_messages = q.receive_messages()
finally:
# Delete the queue
await queue_client.delete_queue()
async def peek_messages_async(self):
# Instantiate a queue client
from azure.storage.queue.aio import QueueClient
queue = QueueClient.from_connection_string(self.connection_string,
"myqueue5")
# Create the queue
async with queue:
await queue.create_queue()
try:
# Send messages
await asyncio.gather(queue.send_message(u"message1"),
queue.send_message(u"message2"),
queue.send_message(u"message3"),
queue.send_message(u"message4"),
queue.send_message(u"message5"))
# [START async_peek_message]
# Peek at one message at the front of the queue
msg = await queue.peek_messages()
# Peek at the last 5 messages
messages = await queue.peek_messages(max_messages=5)
# Print the last 5 messages
for message in messages:
print(message.content)
# [END async_peek_message]
finally:
# Delete the queue
await queue.delete_queue()
async def test_message_base64_decode_fails(self, resource_group, location,
storage_account,
storage_account_key):
# Arrange
qsc = QueueServiceClient(self.account_url(storage_account.name,
"queue"),
storage_account_key,
transport=AiohttpTestTransport())
queue = QueueClient(
account_url=self.account_url(storage_account.name, "queue"),
queue_name=self.get_resource_name(TEST_QUEUE_PREFIX),
credential=storage_account_key,
message_encode_policy=None,
message_decode_policy=BinaryBase64DecodePolicy(),
transport=AiohttpTestTransport())
try:
await queue.create_queue()
except ResourceExistsError:
pass
message = u'xyz'
await queue.send_message(message)
# Action.
with self.assertRaises(DecodeError) as e:
await queue.peek_messages()
# Asserts
self.assertNotEqual(
-1,
str(e.exception).find('Message content is not valid base 64'))
async def test_sas_process(self, resource_group, location, storage_account,
storage_account_key):
qsc = QueueServiceClient(self._account_url(storage_account.name),
storage_account_key,
transport=AiohttpTestTransport())
# SAS URL is calculated from storage key, so this test runs live only
if self.is_playback():
return
# Arrange
queue_client = await self._create_queue(qsc)
await queue_client.enqueue_message(u'message1')
token = queue_client.generate_shared_access_signature(
QueuePermissions.PROCESS,
datetime.utcnow() + timedelta(hours=1),
)
# Act
service = QueueClient(
queue_url=queue_client.url,
credential=token,
)
messages = []
async for m in queue_client.receive_messages():
messages.append(m)
message = messages[0]
# Assert
self.assertIsNotNone(message)
self.assertNotEqual('', message.id)
self.assertEqual(u'message1', message.content)
async def test_sas_read(self, resource_group, location, storage_account,
storage_account_key):
qsc = QueueServiceClient(self._account_url(storage_account.name),
storage_account_key,
transport=AiohttpTestTransport())
# SAS URL is calculated from storage key, so this test runs live only
if self.is_playback():
return
# Arrange
queue_client = await self._create_queue(qsc)
await queue_client.enqueue_message(u'message1')
token = queue_client.generate_shared_access_signature(
QueuePermissions.READ,
datetime.utcnow() + timedelta(hours=1),
datetime.utcnow() - timedelta(minutes=5))
# Act
service = QueueClient(
queue_url=queue_client.url,
credential=token,
)
result = await service.peek_messages()
# Assert
self.assertIsNotNone(result)
self.assertEqual(1, len(result))
message = result[0]
self.assertIsNotNone(message)
self.assertNotEqual('', message.id)
self.assertEqual(u'message1', message.content)
async def test_sas_add(self, resource_group, location, storage_account,
storage_account_key):
qsc = QueueServiceClient(self._account_url(storage_account.name),
storage_account_key,
transport=AiohttpTestTransport())
# SAS URL is calculated from storage key, so this test runs live only
if self.is_playback():
return
# Arrange
queue_client = await self._create_queue(qsc)
token = queue_client.generate_shared_access_signature(
QueueSasPermissions(add=True),
datetime.utcnow() + timedelta(hours=1),
)
# Act
service = QueueClient(
queue_url=queue_client.url,
credential=token,
)
result = await service.enqueue_message(u'addedmessage')
# Assert
messages = []
async for m in queue_client.receive_messages():
messages.append(m)
result = messages[0]
self.assertEqual(u'addedmessage', result.content)
async def _test_sas_signed_identifier(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
access_policy = AccessPolicy()
access_policy.start = datetime.utcnow() - timedelta(hours=1)
access_policy.expiry = datetime.utcnow() + timedelta(hours=1)
access_policy.permission = QueuePermissions.READ
identifiers = {'testid': access_policy}
queue_client = await self._create_queue()
resp = await queue_client.set_queue_access_policy(identifiers)
await queue_client.enqueue_message(u'message1')
token = queue_client.generate_shared_access_signature(
policy_id='testid')
# Act
service = QueueClient(
queue_url=queue_client.url,
credential=token,
)
result = await service.peek_messages()
# Assert
self.assertIsNotNone(result)
self.assertEqual(1, len(result))
message = result[0]
self.assertIsNotNone(message)
self.assertNotEqual('', message.id)
self.assertEqual(u'message1', message.content)
async def _test_sas_process(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
queue_client = await self._create_queue()
await queue_client.enqueue_message(u'message1')
token = queue_client.generate_shared_access_signature(
QueuePermissions.PROCESS,
datetime.utcnow() + timedelta(hours=1),
)
# Act
service = QueueClient(
queue_url=queue_client.url,
credential=token,
)
messages = []
async for m in queue_client.receive_messages():
messages.append(m)
message = messages[0]
# Assert
self.assertIsNotNone(message)
self.assertNotEqual('', message.id)
self.assertEqual(u'message1', message.content)
async def _test_sas_update(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
queue_client = await self._create_queue()
await queue_client.enqueue_message(u'message1')
token = queue_client.generate_shared_access_signature(
QueuePermissions.UPDATE,
datetime.utcnow() + timedelta(hours=1),
)
messages = []
async for m in queue_client.receive_messages():
messages.append(m)
result = messages[0]
# Act
service = QueueClient(
queue_url=queue_client.url,
credential=token,
)
await service.update_message(
result.id,
pop_receipt=result.pop_receipt,
visibility_timeout=0,
content=u'updatedmessage1',
)
# Assert
messages = []
async for m in queue_client.receive_messages():
messages.append(m)
result = messages[0]
self.assertEqual(u'updatedmessage1', result.content)
async def _test_sas_add(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
queue_client = await self._create_queue()
token = queue_client.generate_shared_access_signature(
QueuePermissions.ADD,
datetime.utcnow() + timedelta(hours=1),
)
# Act
service = QueueClient(
queue_url=queue_client.url,
credential=token,
)
result = await service.enqueue_message(u'addedmessage')
# Assert
messages = []
async for m in queue_client.receive_messages():
messages.append(m)
result = messages[0]
self.assertEqual(u'addedmessage', result.content)
async def _test_sas_read(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
queue_client = await self._create_queue()
await queue_client.enqueue_message(u'message1')
token = queue_client.generate_shared_access_signature(
QueuePermissions.READ,
datetime.utcnow() + timedelta(hours=1),
datetime.utcnow() - timedelta(minutes=5))
# Act
service = QueueClient(
queue_url=queue_client.url,
credential=token,
)
result = await service.peek_messages()
# Assert
self.assertIsNotNone(result)
self.assertEqual(1, len(result))
message = result[0]
self.assertIsNotNone(message)
self.assertNotEqual('', message.id)
self.assertEqual(u'message1', message.content)
async def _test_queue_and_messages_example(self):
# Instantiate the QueueClient from a connection string
from azure.storage.queue.aio import QueueClient
queue = QueueClient.from_connection_string(self.connection_string,
"myasyncqueue")
# Create the queue
# [START async_create_queue]
await queue.create_queue()
# [END async_create_queue]
try:
# Enqueue messages
await asyncio.gather(
queue.enqueue_message(u"I'm using queues!"),
queue.enqueue_message(u"This is my second message"))
# Receive the messages
response = queue.receive_messages(messages_per_page=2)
# Print the content of the messages
async for message in response:
print(message.content)
finally:
# [START async_delete_queue]
await queue.delete_queue()
async def receive_one_message_from_queue(self):
# Instantiate a queue client
from azure.storage.queue.aio import QueueClient
queue = QueueClient.from_connection_string(self.connection_string,
"myqueue3")
# Create the queue
async with queue:
await queue.create_queue()
try:
await asyncio.gather(queue.send_message(u"message1"),
queue.send_message(u"message2"),
queue.send_message(u"message3"))
# [START receive_one_message]
# Pop two messages from the front of the queue
message1 = await queue.receive_message()
message2 = await queue.receive_message()
# We should see message 3 if we peek
message3 = await queue.peek_messages()
print(message1.content)
print(message2.content)
print(message3[0].content)
# [END receive_one_message]
finally:
await queue.delete_queue()
async def _test_message_base64_decode_fails(self):
# Arrange
queue_url = self._get_queue_url()
credentials = self._get_shared_key_credential()
queue = QueueClient(queue_url=queue_url,
queue=self.get_resource_name(TEST_QUEUE_PREFIX),
credential=credentials,
message_encode_policy=TextXMLEncodePolicy(),
message_decode_policy=BinaryBase64DecodePolicy(),
transport=AiohttpTestTransport())
try:
await queue.create_queue()
except ResourceExistsError:
pass
message = u'xyz'
await queue.enqueue_message(message)
# Action.
with self.assertRaises(DecodeError) as e:
await queue.peek_messages()
# Asserts
self.assertNotEqual(
-1,
str(e.exception).find('Message content is not valid base 64'))
async def update_message_async(self):
# Instantiate a queue client
from azure.storage.queue.aio import QueueClient
queue = QueueClient.from_connection_string(self.connection_string,
"myqueue6")
# Create the queue
async with queue:
await queue.create_queue()
try:
# [START async_update_message]
# Send a message
await queue.send_message(u"update me")
# Receive the message
messages = queue.receive_messages()
# Update the message
async for message in messages:
message = await queue.update_message(message,
visibility_timeout=0,
content=u"updated")
# [END async_update_message]
break
finally:
# Delete the queue
await queue.delete_queue()
async def test_sas_read(self, resource_group, location, storage_account,
storage_account_key):
qsc = QueueServiceClient(self.account_url(storage_account, "queue"),
storage_account_key,
transport=AiohttpTestTransport())
# SAS URL is calculated from storage key, so this test runs live only
# Arrange
queue_client = await self._create_queue(qsc)
await queue_client.send_message(u'message1')
token = generate_queue_sas(queue_client.account_name,
queue_client.queue_name,
queue_client.credential.account_key,
QueueSasPermissions(read=True),
datetime.utcnow() + timedelta(hours=1),
datetime.utcnow() - timedelta(minutes=5))
# Act
service = QueueClient.from_queue_url(
queue_url=queue_client.url,
credential=token,
)
result = await service.peek_messages()
# Assert
self.assertIsNotNone(result)
self.assertEqual(1, len(result))
message = result[0]
self.assertIsNotNone(message)
self.assertNotEqual('', message.id)
self.assertEqual(u'message1', message.content)
async def test_sas_process(self, resource_group, location, storage_account,
storage_account_key):
qsc = QueueServiceClient(self.account_url(storage_account, "queue"),
storage_account_key,
transport=AiohttpTestTransport())
# SAS URL is calculated from storage key, so this test runs live only
# Arrange
queue_client = await self._create_queue(qsc)
await queue_client.send_message(u'message1')
token = generate_queue_sas(
queue_client.account_name,
queue_client.queue_name,
queue_client.credential.account_key,
QueueSasPermissions(process=True),
datetime.utcnow() + timedelta(hours=1),
)
# Act
service = QueueClient.from_queue_url(
queue_url=queue_client.url,
credential=token,
)
messages = []
async for m in queue_client.receive_messages():
messages.append(m)
message = messages[0]
# Assert
self.assertIsNotNone(message)
self.assertNotEqual('', message.id)
self.assertEqual(u'message1', message.content)
async def test_update_message(self, resource_group, location,
storage_account, storage_account_key):
# Instantiate a queue client
from azure.storage.queue.aio import QueueClient
queue = QueueClient.from_connection_string(
self.connection_string(storage_account, storage_account_key),
"asyncupdatequeue")
# Create the queue
await queue.create_queue()
try:
# [START async_update_message]
# Send a message
await queue.send_message(u"update me")
# Receive the message
messages = queue.receive_messages()
# Update the message
async for message in messages:
message = await queue.update_message(message,
visibility_timeout=0,
content=u"updated")
# [END async_update_message]
assert message.content == "updated"
break
finally:
# Delete the queue
await queue.delete_queue()
async def test_queue_metadata(self, resource_group, location,
storage_account, storage_account_key):
# Instantiate a queue client
from azure.storage.queue.aio import QueueClient
queue = QueueClient.from_connection_string(
self.connection_string(storage_account, storage_account_key),
"asyncmetaqueue")
# Create the queue
await queue.create_queue()
try:
# [START async_set_queue_metadata]
metadata = {'foo': 'val1', 'bar': 'val2', 'baz': 'val3'}
await queue.set_queue_metadata(metadata=metadata)
# [END async_set_queue_metadata]
# [START async_get_queue_properties]
response = await queue.get_queue_properties()
# [END async_get_queue_properties]
assert response.metadata == metadata
finally:
# Delete the queue
await queue.delete_queue()
async def _test_update_message(self):
# Instantiate a queue client
from azure.storage.queue.aio import QueueClient
queue = QueueClient.from_connection_string(self.connection_string,
"asyncupdatequeue")
# Create the queue
await queue.create_queue()
try:
# [START async_update_message]
# Enqueue a message
await queue.enqueue_message(u"update me")
# Receive the message
messages = queue.receive_messages()
# Update the message
async for message in messages:
message = await queue.update_message(message,
visibility_timeout=0,
content=u"updated")
# [END update_message]
assert message.content == "updated"
break
finally:
# Delete the queue
await queue.delete_queue()
async def test_peek_messages(self, resource_group, location,
storage_account, storage_account_key):
# Instantiate a queue client
from azure.storage.queue.aio import QueueClient
queue = QueueClient.from_connection_string(
self.connection_string(storage_account, storage_account_key),
"asyncpeekqueue")
# Create the queue
await queue.create_queue()
try:
# Send messages
await asyncio.gather(queue.send_message(u"message1"),
queue.send_message(u"message2"),
queue.send_message(u"message3"),
queue.send_message(u"message4"),
queue.send_message(u"message5"))
# [START async_peek_message]
# Peek at one message at the front of the queue
msg = await queue.peek_messages()
# Peek at the last 5 messages
messages = await queue.peek_messages(max_messages=5)
# Print the last 5 messages
for message in messages:
print(message.content)
# [END async_peek_message]
finally:
# Delete the queue
await queue.delete_queue()
async def _test_delete_and_clear_messages(self):
# Instantiate a queue client
from azure.storage.queue.aio import QueueClient
queue = QueueClient.from_connection_string(self.connection_string,
"asyncdelqueue")
# Create the queue
await queue.create_queue()
try:
# Enqueue messages
await asyncio.gather(queue.enqueue_message(u"message1"),
queue.enqueue_message(u"message2"),
queue.enqueue_message(u"message3"),
queue.enqueue_message(u"message4"),
queue.enqueue_message(u"message5"))
# [START async_delete_message]
# Get the message at the front of the queue
messages = queue.receive_messages()
async for msg in messages:
# Delete the specified message
await queue.delete_message(msg)
# [END async_delete_message]
# [START async_clear_messages]
await queue.clear_messages()
# [END async_clear_messages]
finally:
# Delete the queue
await queue.delete_queue()
async def set_access_policy_async(self):
# [START async_create_queue_client_from_connection_string]
from azure.storage.queue.aio import QueueClient
queue = QueueClient.from_connection_string(self.connection_string,
"myqueue1")
# [END async_create_queue_client_from_connection_string]
# Create the queue
async with queue:
await queue.create_queue()
# Send a message
await queue.send_message(u"hello world")
try:
# [START async_set_access_policy]
# Create an access policy
from azure.storage.queue import AccessPolicy, QueueSasPermissions
access_policy = AccessPolicy()
access_policy.start = datetime.utcnow() - timedelta(hours=1)
access_policy.expiry = datetime.utcnow() + timedelta(hours=1)
access_policy.permission = QueueSasPermissions(read=True)
identifiers = {'my-access-policy-id': access_policy}
# Set the access policy
await queue.set_queue_access_policy(identifiers)
# [END async_set_access_policy]
# Use the access policy to generate a SAS token
from azure.storage.queue import generate_queue_sas
sas_token = generate_queue_sas(queue.account_name,
queue.queue_name,
queue.credential.account_key,
policy_id='my-access-policy-id')
# Authenticate with the sas token
# [START async_create_queue_client]
token_auth_queue = QueueClient.from_queue_url(
queue_url=queue.url, credential=sas_token)
# [END async_create_queue_client]
# Use the newly authenticated client to receive messages
my_messages = token_auth_queue.receive_messages()
finally:
# Delete the queue
await queue.delete_queue()
def test_queue_client_api_version_property(self):
queue_client = QueueClient(
"https://foo.queue.core.windows.net/account",
"queue_name",
credential="fake_key",
api_version=self.api_version_1)
self.assertEqual(queue_client.api_version, self.api_version_1)
self.assertEqual(queue_client._client._config.version,
self.api_version_1)
queue_client = QueueClient(
"https://foo.queue.core.windows.net/account",
"queue_name",
credential="fake_key")
self.assertEqual(queue_client.api_version, self.api_version_2)
self.assertEqual(queue_client._client._config.version,
self.api_version_2)
async def _test_set_access_policy(self):
# [START async_create_queue_client_from_connection_string]
from azure.storage.queue.aio import QueueClient
queue_client = QueueClient.from_connection_string(
self.connection_string, "asyncqueuetest")
# [END async_create_queue_client_from_connection_string]
# Create the queue
try:
await queue_client.create_queue()
except ResourceExistsError:
pass
await queue_client.enqueue_message(u"hello world")
try:
# [START async_set_access_policy]
# Create an access policy
from azure.storage.queue.aio import AccessPolicy, QueuePermissions
access_policy = AccessPolicy()
access_policy.start = datetime.utcnow() - timedelta(hours=1)
access_policy.expiry = datetime.utcnow() + timedelta(hours=1)
access_policy.permission = QueuePermissions.READ
identifiers = {'my-access-policy-id': access_policy}
# Set the access policy
await queue_client.set_queue_access_policy(identifiers)
# [END async_set_access_policy]
# Use the access policy to generate a SAS token
sas_token = queue_client.generate_shared_access_signature(
policy_id='my-access-policy-id')
# [END async_set_access_policy]
# Authenticate with the sas token
# [START async_create_queue_client]
from azure.storage.queue.aio import QueueClient
q = QueueClient(queue_url=queue_client.url, credential=sas_token)
# [END async_create_queue_client]
# Use the newly authenticated client to receive messages
my_messages = q.receive_messages()
finally:
# Delete the queue
await queue_client.delete_queue()
def get_queue_client(url: str) -> QueueClient:
""" Initialize queue client
"""
queue_client_dict = dict()
if not queue_client_dict.get(url):
client = QueueClient.from_queue_url(url, credential=ADX_INGEST_QUEUE_SAS_TOKEN)
queue_client_dict[url] = client
logging.info(f"{HEADER} Initialize Queue Client for {url}")
return queue_client_dict[url]
def __init__(self, queues_storage_connection_string: str, queue_name: str):
if not queues_storage_connection_string:
raise Exception(
"queues_storage_connection_string cannot be empty.")
if not queue_name:
raise Exception("queue_name cannot be empty.")
self.__queue_client = QueueClient.from_connection_string(
queues_storage_connection_string, queue_name)
self.__initialized = False
async def log_responses(self, responses):
if self.credential == 'default' or self.queue_url == 'default':
for response in responses:
print(response)
return
else:
async with QueueClient.from_queue_url(
queue_url=self.queue_url,
credential=self.credential) as async_storage_queue:
for response in responses:
await async_storage_queue.send_message(str(response))
async def _test_message_bytes_base64(self):
# Arrange.
queue_url = self._get_queue_url()
credentials = self._get_shared_key_credential()
queue = QueueClient(queue_url=queue_url,
queue=self.get_resource_name(TEST_QUEUE_PREFIX),
credential=credentials,
message_encode_policy=BinaryBase64EncodePolicy(),
message_decode_policy=BinaryBase64DecodePolicy(),
transport=AiohttpTestTransport())
message = b'xyz'
# Asserts
await self._validate_encoding(queue, message)
