def test_update_encrypted_binary_message(self, resource_group, location, storage_account, storage_account_key): # Arrange qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key) queue = self._create_queue( qsc, message_encode_policy=BinaryBase64EncodePolicy(), message_decode_policy=BinaryBase64DecodePolicy()) queue.key_encryption_key = KeyWrapper('key1') binary_message = self.get_random_bytes(100) queue.send_message(binary_message) messages = [] for m in queue.receive_messages(): messages.append(m) list_result1 = messages[0] # Act binary_message = self.get_random_bytes(100) list_result1.content = binary_message queue.update_message(list_result1) for m in queue.receive_messages(): messages.append(m) list_result2 = messages[0] # Assert self.assertEqual(binary_message, list_result2.content)
async def _test_update_encrypted_binary_message(self): # Arrange queue = await self._create_queue() queue.key_encryption_key = KeyWrapper('key1') queue._config.message_encode_policy = BinaryBase64EncodePolicy() queue._config.message_decode_policy = BinaryBase64DecodePolicy() binary_message = self.get_random_bytes(100) await queue.enqueue_message(binary_message) messages = [] async for m in queue.receive_messages(): messages.append(m) list_result1 = messages[0] # Act binary_message = self.get_random_bytes(100) list_result1.content = binary_message await queue.update_message(list_result1) async for m in queue.receive_messages(): messages.append(m) list_result2 = messages[0] # Assert self.assertEqual(binary_message, list_result2.content)
async def test_update_encrypted_binary_message(self, storage_account_name, storage_account_key): qsc = QueueServiceClient(self.account_url(storage_account_name, "queue"), storage_account_key, transport=AiohttpTestTransport()) # Arrange queue = await self._create_queue( qsc, message_encode_policy=BinaryBase64EncodePolicy(), message_decode_policy=BinaryBase64DecodePolicy()) queue.key_encryption_key = KeyWrapper('key1') binary_message = self.get_random_bytes(100) await queue.send_message(binary_message) messages = [] async for m in queue.receive_messages(): messages.append(m) list_result1 = messages[0] # Act binary_message = self.get_random_bytes(100) list_result1.content = binary_message await queue.update_message(list_result1) async for m in queue.receive_messages(): messages.append(m) list_result2 = messages[0] # Assert self.assertEqual(binary_message, list_result2.content)
def test_message_base64_decode_fails(self, resource_group, location, storage_account, storage_account_key): # Arrange qsc = QueueServiceClient(self._account_url(storage_account.name), storage_account_key) queue = QueueClient( account_url=self._account_url(storage_account.name), queue_name=self.get_resource_name(TEST_QUEUE_PREFIX), credential=storage_account_key, message_encode_policy=None, message_decode_policy=BinaryBase64DecodePolicy()) try: queue.create_queue() except ResourceExistsError: pass message = u'xyz' queue.send_message(message) # Action. with self.assertRaises(DecodeError) as e: queue.peek_messages() # Asserts self.assertNotEqual( -1, str(e.exception).find('Message content is not valid base 64'))
async def _test_message_base64_decode_fails(self): # Arrange queue_url = self._get_queue_url() credentials = self._get_shared_key_credential() queue = QueueClient(queue_url=queue_url, queue=self.get_resource_name(TEST_QUEUE_PREFIX), credential=credentials, message_encode_policy=TextXMLEncodePolicy(), message_decode_policy=BinaryBase64DecodePolicy(), transport=AiohttpTestTransport()) try: await queue.create_queue() except ResourceExistsError: pass message = u'xyz' await queue.enqueue_message(message) # Action. with self.assertRaises(DecodeError) as e: await queue.peek_messages() # Asserts self.assertNotEqual( -1, str(e.exception).find('Message content is not valid base 64'))
def send_chunk_info(self, export_job_id, chunk_id): with QueueClient.from_connection_string( self.connection_string, self.queue_name, message_encode_policy=BinaryBase64EncodePolicy(), message_decode_policy=BinaryBase64DecodePolicy( )) as queue_client: chunk_info = {'exportJobId': export_job_id, 'chunkId': chunk_id} return queue_client.send_message( json.dumps(chunk_info).encode('utf-8'))
def create(self): with QueueClient.from_connection_string( self.connection_string, self.queue_name, message_encode_policy=BinaryBase64EncodePolicy(), message_decode_policy=BinaryBase64DecodePolicy( )) as queue_client: try: queue_client.create_queue() except ResourceExistsError: logging.warn(f'Queue {self.queue_name} already exists')
def test_message_bytes_base64(self, storage_account_name, storage_account_key): # Arrange. qsc = QueueServiceClient(self.account_url(storage_account_name, "queue"), storage_account_key) queue = QueueClient( account_url=self.account_url(storage_account_name, "queue"), queue_name=self.get_resource_name(TEST_QUEUE_PREFIX), credential=storage_account_key, message_encode_policy=BinaryBase64EncodePolicy(), message_decode_policy=BinaryBase64DecodePolicy()) message = b'xyz' # Asserts self._validate_encoding(queue, message)
def __init__(self, queue_name, queue_connection_string=None): """Initialize a JobQueue object.""" logger.debug("Initializing a JobQueue for [%s]", queue_name) if queue_connection_string is None: queue_connection_string = DEFAULT_QUEUE_CONNECTION_STRING self._client = QueueClient.from_connection_string( queue_connection_string, queue_name) self._client.message_encode_policy = BinaryBase64EncodePolicy() self._client.message_decode_policy = BinaryBase64DecodePolicy() try: self._client.create_queue() except ResourceExistsError: pass # OK, just means the queue already exists.
def test_message_bytes_base64(self): # Arrange. queue_url = self._get_queue_url() credentials = self._get_shared_key_credential() queue = QueueClient( queue_url=queue_url, queue=self.get_resource_name(TEST_QUEUE_PREFIX), credential=credentials, message_encode_policy=BinaryBase64EncodePolicy(), message_decode_policy=BinaryBase64DecodePolicy()) message = b'xyz' # Asserts self._validate_encoding(queue, message)
def get_messages(connection_string, queue_name): while True: queue_client = QueueClient.from_connection_string( connection_string, queue_name) queue_client.message_encode_policy = BinaryBase64EncodePolicy() queue_client.message_decode_policy = BinaryBase64DecodePolicy() messages = queue_client.receive_messages() for message in messages: print("\nMessage Received: {}".format( base64.b64decode(message.content).decode('utf-8'))) queue_client.delete_message(message.id, message.pop_receipt) properties = queue_client.get_queue_properties() if properties.approximate_message_count == 0: time.sleep(1)
async def test_message_bytes_base64(self, resource_group, location, storage_account, storage_account_key): # Arrange. qsc = QueueServiceClient(self._account_url(storage_account.name), storage_account_key, transport=AiohttpTestTransport()) queue = QueueClient(queue_url=self._account_url(storage_account.name), queue=self.get_resource_name(TEST_QUEUE_PREFIX), credential=storage_account_key, message_encode_policy=BinaryBase64EncodePolicy(), message_decode_policy=BinaryBase64DecodePolicy(), transport=AiohttpTestTransport()) message = b'xyz' # Asserts await self._validate_encoding(queue, message)
def test_message_text_fails(self, storage_account_name, storage_account_key): # Arrange qsc = QueueServiceClient(self.account_url(storage_account_name, "queue"), storage_account_key) queue = QueueClient( account_url=self.account_url(storage_account_name, "queue"), queue_name=self.get_resource_name(TEST_QUEUE_PREFIX), credential=storage_account_key, message_encode_policy=BinaryBase64EncodePolicy(), message_decode_policy=BinaryBase64DecodePolicy()) # Action. with self.assertRaises(TypeError) as e: message = u'xyz' queue.send_message(message) # Asserts self.assertTrue(str(e.exception).startswith('Message content must be bytes'))
def initialize_queue(connection_string: str, queue_name: str) -> QueueClient: """Initialize a queue given the connection string and queue name.""" if connection_string is None or queue_name is None: return None client = None try: client = QueueClient.from_connection_string(connection_string, queue_name) client.message_encode_policy = BinaryBase64EncodePolicy() client.message_decode_policy = BinaryBase64DecodePolicy() try: client.create_queue() except: pass # OK to ignore except: client = None return client
def test_message_text_fails(self): # Arrange queue_url = self._get_queue_url() credentials = self._get_shared_key_credential() queue = QueueClient( queue_url=queue_url, queue=self.get_resource_name(TEST_QUEUE_PREFIX), credential=credentials, message_encode_policy=BinaryBase64EncodePolicy(), message_decode_policy=BinaryBase64DecodePolicy()) # Action. with self.assertRaises(TypeError) as e: message = u'xyz' queue.enqueue_message(message) # Asserts self.assertTrue(str(e.exception).startswith('Message content must be bytes'))
async def test_message_text_fails(self, resource_group, location, storage_account, storage_account_key): # Arrange qsc = QueueServiceClient(self._account_url(storage_account.name), storage_account_key, transport=AiohttpTestTransport()) queue = QueueClient(queue_url=self._account_url(storage_account.name), queue=self.get_resource_name(TEST_QUEUE_PREFIX), credential=storage_account_key, message_encode_policy=BinaryBase64EncodePolicy(), message_decode_policy=BinaryBase64DecodePolicy(), transport=AiohttpTestTransport()) # Action. with self.assertRaises(TypeError) as e: message = u'xyz' await queue.enqueue_message(message) # Asserts self.assertTrue( str(e.exception).startswith('Message content must be bytes'))
def test_update_encrypted_binary_message(self): # Arrange queue = self._create_queue() queue.key_encryption_key = KeyWrapper('key1') queue._config.message_encode_policy = BinaryBase64EncodePolicy() queue._config.message_decode_policy = BinaryBase64DecodePolicy() binary_message = self.get_random_bytes(100) queue.enqueue_message(binary_message) messages = queue.receive_messages() list_result1 = next(messages) # Act binary_message = self.get_random_bytes(100) list_result1.content = binary_message queue.update_message(list_result1) list_result2 = next(messages) # Assert self.assertEqual(binary_message, list_result2.content)
def step3(): with open('../local.settings.json') as fd: settings = json.load(fd) connectionString = settings["Values"]["AzureWebJobsStorage"] #os.environ["AZURE_STORAGE_CONNECTION_STRING"] = connectionString container_name = "opengameart" with open('putToSqlQueue.txt') as fd: files = [fname.strip() for fname in fd.readlines()] #files = [fname for fname in files if fname.lower().endswith('.jpg') or fname.lower().endswith('.png')] files = files[282000:] # Retrieve the connection string from an environment # variable named AZURE_STORAGE_CONNECTION_STRING connect_str = os.getenv("AZURE_STORAGE_CONNECTION_STRING") # Create a unique name for the queue #q_name = "queue-" + str(uuid.uuid4()) q_name = 'sqlqueue' # Instantiate a QueueClient object which will # be used to create and manipulate the queue #queue_client = QueueClient.from_connection_string(connectionString, q_name) # Setup Base64 encoding and decoding functions base64_queue_client = QueueClient.from_connection_string( conn_str=connectionString, queue_name=q_name, message_encode_policy = BinaryBase64EncodePolicy(), message_decode_policy = BinaryBase64DecodePolicy() ) for i, message in enumerate(files): if i % 1000 == 0: print(i) base64_queue_client.send_message(message.encode('ascii'))
FILE: consume_cloud_events_from_storage_queue.py DESCRIPTION: These samples demonstrate receiving events from a Storage Queue. USAGE: python consume_cloud_events_from_storage_queue.py Set the environment variables with your own values before running the sample: 1) STORAGE_QUEUE_CONN_STR: The connection string to the Storage account 3) STORAGE_QUEUE_NAME: The name of the storage queue. """ from azure.eventgrid import CloudEvent from azure.storage.queue import QueueServiceClient, BinaryBase64DecodePolicy import os import json # all types of CloudEvents below produce same DeserializedEvent connection_str = os.environ['STORAGE_QUEUE_CONN_STR'] queue_name = os.environ['STORAGE_QUEUE_NAME'] with QueueServiceClient.from_connection_string(connection_str) as qsc: payload = qsc.get_queue_client( queue=queue_name, message_decode_policy=BinaryBase64DecodePolicy() ).peek_messages() ## deserialize payload into a lost of typed Events events = [CloudEvent.from_dict(json.loads(msg.content)) for msg in payload] for event in events: print(type(event)) ## CloudEvent
""" FILE: consume_cloud_events_from_storage_queue.py DESCRIPTION: These samples demonstrate receiving events from a Storage Queue. USAGE: python consume_cloud_events_from_storage_queue.py Set the environment variables with your own values before running the sample: 1) STORAGE_QUEUE_CONN_STR: The connection string to the Storage account 3) STORAGE_QUEUE_NAME: The name of the storage queue. """ from azure.core.messaging import CloudEvent from azure.storage.queue import QueueServiceClient, BinaryBase64DecodePolicy import os import json # all types of CloudEvents below produce same DeserializedEvent connection_str = os.environ['AZURE_STORAGE_CONNECTION_STRING'] queue_name = os.environ['STORAGE_QUEUE_NAME'] with QueueServiceClient.from_connection_string(connection_str) as qsc: payload = qsc.get_queue_client( queue=queue_name, message_decode_policy=BinaryBase64DecodePolicy()).peek_messages() ## deserialize payload into a list of typed Events events = [CloudEvent.from_dict(json.loads(msg.content)) for msg in payload] for event in events: print(type(event)) ## CloudEvent
# Info for authenticationg # Display name : huyeduon-restapi # Application (client) ID : c4f505e2-b31b-42ad-8294-11991044cece # Directory (tenant) ID : e34b6079-b8db-43a4-8104-4768856bd06c # Object ID : 52a4d658-867e-41fa-947d-2f51befd0f2d tenant_id = "e34b6079-b8db-43a4-8104-4768856bd06c" client_id = "c4f505e2-b31b-42ad-8294-11991044cece" client_secret = "E9LpKrW-rqFabdc3-2aMSML4Hq~-k.R.rw" # Get the application credentials app_credentials = ClientSecretCredential(tenant_id, client_id, client_secret) # Create a queue client queue_client = QueueClient.from_connection_string(connect_str, processing_queue_name, credential=app_credentials) # Setup Base64 encoding and decoding functions queue_client.message_encode_policy = BinaryBase64EncodePolicy() queue_client.message_decode_policy = BinaryBase64DecodePolicy() print("Client connected to Queue") print("Peek messages in the queue...") messages = queue_client.peek_messages(max_messages=5) for peeked_message in messages: print("Message: ", base64ToString(peeked_message.content))