def test_account_sas(self, resource_group, location, storage_account, storage_account_key): # SAS URL is calculated from storage key, so this test runs live only # Arrange qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key) queue_client = self._get_queue_reference(qsc) queue_client.create_queue() queue_client.send_message(u'message1') token = generate_account_sas( qsc.account_name, qsc.credential.account_key, ResourceTypes(object=True), AccountSasPermissions(read=True), datetime.utcnow() + timedelta(hours=1), datetime.utcnow() - timedelta(minutes=5) ) # Act service = QueueServiceClient( account_url=qsc.url, credential=token, ) new_queue_client = service.get_queue_client(queue_client.queue_name) result = new_queue_client.peek_messages() # Assert self.assertIsNotNone(result) self.assertEqual(1, len(result)) message = result[0] self.assertIsNotNone(message) self.assertNotEqual('', message.id) self.assertEqual(u'message1', message.content)
def test_account_sas(self, resource_group, location, storage_account, storage_account_key): # SAS URL is calculated from storage key, so this test runs live only if not self.is_live: return # Arrange qsc = QueueServiceClient(self._account_url(storage_account.name), storage_account_key) queue_client = self._get_queue_reference(qsc) queue_client.create_queue() queue_client.enqueue_message(u'message1') token = qsc.generate_shared_access_signature( ResourceTypes.OBJECT, AccountPermissions.READ, datetime.utcnow() + timedelta(hours=1), datetime.utcnow() - timedelta(minutes=5)) # Act service = QueueServiceClient( account_url=qsc.url, credential=token, ) new_queue_client = service.get_queue_client(queue_client.queue_name) result = new_queue_client.peek_messages() # Assert self.assertIsNotNone(result) self.assertEqual(1, len(result)) message = result[0] self.assertIsNotNone(message) self.assertNotEqual('', message.id) self.assertEqual(u'message1', message.content)
def test_token_credential(self, resource_group, location, storage_account, storage_account_key): token_credential = self.generate_oauth_token() # Action 1: make sure token works service = QueueServiceClient(self.account_url(storage_account, "queue"), credential=token_credential) queues = service.get_service_properties() self.assertIsNotNone(queues) # Action 2: change token value to make request fail fake_credential = self.generate_fake_token() service = QueueServiceClient(self.account_url(storage_account, "queue"), credential=fake_credential) with self.assertRaises(ClientAuthenticationError): list(service.list_queues()) # Action 3: update token to make it working again service = QueueServiceClient(self.account_url(storage_account, "queue"), credential=token_credential) queues = list(service.list_queues()) self.assertIsNotNone(queues)
def test_update_encrypted_binary_message(self, resource_group, location, storage_account, storage_account_key): # Arrange qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key) queue = self._create_queue( qsc, message_encode_policy=BinaryBase64EncodePolicy(), message_decode_policy=BinaryBase64DecodePolicy()) queue.key_encryption_key = KeyWrapper('key1') binary_message = self.get_random_bytes(100) queue.send_message(binary_message) messages = [] for m in queue.receive_messages(): messages.append(m) list_result1 = messages[0] # Act binary_message = self.get_random_bytes(100) list_result1.content = binary_message queue.update_message(list_result1) for m in queue.receive_messages(): messages.append(m) list_result2 = messages[0] # Assert self.assertEqual(binary_message, list_result2.content)
def setUp(self): super(StorageQueueTest, self).setUp() queue_url = self._get_queue_url() credentials = self._get_shared_key_credential() self.qsc = QueueServiceClient(account_url=queue_url, credential=credentials) self.test_queues = []
def test_account_sas_raises_if_sas_already_in_uri(self, storage_account_name, storage_account_key): with self.assertRaises(ValueError): QueueServiceClient( self.account_url(storage_account_name, "queue") + "?sig=foo", credential=AzureSasCredential("?foo=bar"))
def test_peek_messages_with_options(self, resource_group, location, storage_account, storage_account_key): # Action qsc = QueueServiceClient(self._account_url(storage_account.name), storage_account_key) queue_client = self._get_queue_reference(qsc) queue_client.create_queue() queue_client.enqueue_message(u'message1') queue_client.enqueue_message(u'message2') queue_client.enqueue_message(u'message3') queue_client.enqueue_message(u'message4') result = queue_client.peek_messages(max_messages=4) # Asserts self.assertIsNotNone(result) self.assertEqual(4, len(result)) for message in result: self.assertIsNotNone(message) self.assertNotEqual('', message.id) self.assertNotEqual('', message.content) self.assertIsNone(message.pop_receipt) self.assertEqual(0, message.dequeue_count) self.assertNotEqual('', message.insertion_time) self.assertNotEqual('', message.expiration_time) self.assertIsNone(message.time_next_visible)
def test_account_sas(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # Arrange queue_client = self._create_queue() queue_client.enqueue_message(u'message1') token = self.qsc.generate_shared_access_signature( ResourceTypes.OBJECT, AccountPermissions.READ, datetime.utcnow() + timedelta(hours=1), datetime.utcnow() - timedelta(minutes=5)) # Act service = QueueServiceClient( account_url=self.qsc.url, credential=token, ) new_queue_client = service.get_queue_client(queue_client.queue_name) result = new_queue_client.peek_messages() # Assert self.assertIsNotNone(result) self.assertEqual(1, len(result)) message = result[0] self.assertIsNotNone(message) self.assertNotEqual('', message.id) self.assertEqual(u'message1', message.content)
def test_missing_attribute_kek_unrwap(self, resource_group, location, storage_account, storage_account_key): # Arrange qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key) queue = self._create_queue(qsc) queue.key_encryption_key = KeyWrapper('key1') queue.send_message(u'message') # Act valid_key = KeyWrapper('key1') invalid_key_1 = lambda: None # functions are objects, so this effectively creates an empty object invalid_key_1.unwrap_key = valid_key.unwrap_key # No attribute get_kid queue.key_encryption_key = invalid_key_1 with self.assertRaises(HttpResponseError) as e: queue.peek_messages() self.assertEqual(str(e.exception), "Decryption failed.") invalid_key_2 = lambda: None # functions are objects, so this effectively creates an empty object invalid_key_2.get_kid = valid_key.get_kid # No attribute unwrap_key queue.key_encryption_key = invalid_key_2 with self.assertRaises(HttpResponseError): queue.peek_messages()
def test_invalid_value_kek_wrap(self, resource_group, location, storage_account, storage_account_key): # Arrange qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key) queue = self._create_queue(qsc) queue.key_encryption_key = KeyWrapper('key1') queue.key_encryption_key.get_kid = None with self.assertRaises(AttributeError) as e: queue.send_message(u'message') self.assertEqual( str(e.exception), _ERROR_OBJECT_INVALID.format('key encryption key', 'get_kid')) queue.key_encryption_key = KeyWrapper('key1') queue.key_encryption_key.get_kid = None with self.assertRaises(AttributeError): queue.send_message(u'message') queue.key_encryption_key = KeyWrapper('key1') queue.key_encryption_key.wrap_key = None with self.assertRaises(AttributeError): queue.send_message(u'message')
def test_queue_not_exists(self, resource_group, location, storage_account, storage_account_key): # Arrange qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key) queue = qsc.get_queue_client(self.get_resource_name('missing')) # Act with self.assertRaises(ResourceNotFoundError): queue.get_queue_properties()
def test_list_queues_with_options(self, resource_group, location, storage_account, storage_account_key): # Arrange prefix = 'listqueue' qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key) queue_list = [] for i in range(0, 4): self._create_queue(qsc, prefix + str(i), queue_list) # Action generator1 = qsc.list_queues( name_starts_with=prefix, results_per_page=3).by_page() queues1 = list(next(generator1)) generator2 = qsc.list_queues( name_starts_with=prefix, include_metadata=True).by_page(generator1.continuation_token) queues2 = list(next(generator2)) # Asserts self.assertIsNotNone(queues1) self.assertEqual(3, len(queues1)) self.assertIsNotNone(queues1[0]) self.assertIsNone(queues1[0].metadata) self.assertNotEqual('', queues1[0].name) assert generator1.location_mode is not None # Asserts self.assertIsNotNone(queues2) self.assertTrue(len(queue_list) - 3 <= len(queues2)) self.assertIsNotNone(queues2[0]) self.assertNotEqual('', queues2[0].name)
def test_unicode_update_message_unicode_data(self, resource_group, location, storage_account, storage_account_key): # Action pytest.skip("Uncomment after msrest fix") qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key) queue_client = self._get_queue_reference(qsc) queue_client.create_queue() queue_client.send_message(u'message1') messages = queue_client.receive_messages() list_result1 = next(messages) list_result1.content = u'啊齄丂狛狜' queue_client.update_message(list_result1, visibility_timeout=0) # Asserts message = next(messages) self.assertIsNotNone(message) self.assertEqual(list_result1.id, message.id) self.assertEqual(u'啊齄丂狛狜', message.content) self.assertNotEqual('', message.pop_receipt) self.assertEqual(2, message.dequeue_count) self.assertIsInstance(message.inserted_on, datetime) self.assertIsInstance(message.expires_on, datetime) self.assertIsInstance(message.next_visible_on, datetime)
def test_set_queue_acl_with_signed_identifiers(self, resource_group, location, storage_account, storage_account_key): # Arrange qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key) queue_client = self._get_queue_reference(qsc) queue_client.create_queue() # Act access_policy = AccessPolicy( permission=QueueSasPermissions(read=True), expiry=datetime.utcnow() + timedelta(hours=1), start=datetime.utcnow() - timedelta(minutes=5)) identifiers = {'testid': access_policy} resp = queue_client.set_queue_access_policy( signed_identifiers=identifiers) # Assert self.assertIsNone(resp) acl = queue_client.get_queue_access_policy() self.assertIsNotNone(acl) self.assertEqual(len(acl), 1) self.assertTrue('testid' in acl)
def test_peek_messages(self, resource_group, location, storage_account, storage_account_key): # Action qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key) queue_client = self._get_queue_reference(qsc) queue_client.create_queue() queue_client.send_message(u'message1') queue_client.send_message(u'message2') queue_client.send_message(u'message3') queue_client.send_message(u'message4') result = queue_client.peek_messages() # Asserts self.assertIsNotNone(result) self.assertEqual(1, len(result)) message = result[0] self.assertIsNotNone(message) self.assertNotEqual('', message.id) self.assertNotEqual('', message.content) self.assertIsNone(message.pop_receipt) self.assertEqual(0, message.dequeue_count) self.assertNotEqual('', message.inserted_on) self.assertNotEqual('', message.expires_on) self.assertIsNone(message.next_visible_on)
def test_sas_add(self, resource_group, location, storage_account, storage_account_key): # SAS URL is calculated from storage key, so this test runs live only if not self.is_live: return # Arrange qsc = QueueServiceClient(self._account_url(storage_account.name), storage_account_key) queue_client = self._get_queue_reference(qsc) queue_client.create_queue() token = queue_client.generate_shared_access_signature( QueueSasPermissions(add=True), datetime.utcnow() + timedelta(hours=1), ) # Act service = QueueClient( queue_url=queue_client.url, credential=token, ) result = service.enqueue_message(u'addedmessage') # Assert result = next(queue_client.receive_messages()) self.assertEqual(u'addedmessage', result.content)
def test_update_encrypted_raw_text_message(self, resource_group, location, storage_account, storage_account_key): # TODO: Recording doesn't work # Arrange qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key) queue = self._create_queue(qsc, message_encode_policy=None, message_decode_policy=None) queue.key_encryption_key = KeyWrapper('key1') raw_text = u'Update Me' queue.send_message(raw_text) messages = queue.receive_messages() list_result1 = next(messages) # Act raw_text = u'Updated' list_result1.content = raw_text queue.update_message(list_result1) list_result2 = next(messages) # Assert self.assertEqual(raw_text, list_result2.content)
def test_get_messages_with_options(self, resource_group, location, storage_account, storage_account_key): # Action qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key) queue_client = self._get_queue_reference(qsc) queue_client.create_queue() queue_client.send_message(u'message1') queue_client.send_message(u'message2') queue_client.send_message(u'message3') queue_client.send_message(u'message4') pager = queue_client.receive_messages(messages_per_page=4, visibility_timeout=20) result = list(pager) # Asserts self.assertIsNotNone(result) self.assertEqual(4, len(result)) for message in result: self.assertIsNotNone(message) self.assertNotEqual('', message.id) self.assertNotEqual('', message.content) self.assertNotEqual('', message.pop_receipt) self.assertEqual(1, message.dequeue_count) self.assertNotEqual('', message.inserted_on) self.assertNotEqual('', message.expires_on) self.assertNotEqual('', message.next_visible_on)
def test_update_encrypted_json_message(self, resource_group, location, storage_account, storage_account_key): # TODO: Recording doesn't work # Arrange qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key) queue = self._create_queue(qsc, message_encode_policy=None, message_decode_policy=None) queue.key_encryption_key = KeyWrapper('key1') message_dict = {'val1': 1, 'val2': '2'} json_text = dumps(message_dict) queue.send_message(json_text) messages = queue.receive_messages() list_result1 = next(messages) # Act message_dict['val1'] = 0 message_dict['val2'] = 'updated' json_text = dumps(message_dict) list_result1.content = json_text queue.update_message(list_result1) list_result2 = next(messages) # Assert self.assertEqual(message_dict, loads(list_result2.content))
def test_update_message_content(self, resource_group, location, storage_account, storage_account_key): # Action qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key) queue_client = self._get_queue_reference(qsc) queue_client.create_queue() queue_client.send_message(u'message1') messages = queue_client.receive_messages() list_result1 = next(messages) message = queue_client.update_message( list_result1.id, pop_receipt=list_result1.pop_receipt, visibility_timeout=0, content=u'new text') list_result2 = next(messages) # Asserts # Update response self.assertIsNotNone(message) self.assertIsNotNone(message.pop_receipt) self.assertIsNotNone(message.next_visible_on) self.assertIsInstance(message.next_visible_on, datetime) self.assertEqual(u'new text', message.content) # Get response self.assertIsNotNone(list_result2) message = list_result2 self.assertIsNotNone(message) self.assertEqual(list_result1.id, message.id) self.assertEqual(u'new text', message.content) self.assertEqual(2, message.dequeue_count) self.assertIsNotNone(message.pop_receipt) self.assertIsNotNone(message.inserted_on) self.assertIsNotNone(message.expires_on) self.assertIsNotNone(message.next_visible_on)
def test_missing_attribute_kek_wrap(self, resource_group, location, storage_account, storage_account_key): # Arrange qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key) queue = self._create_queue(qsc) valid_key = KeyWrapper('key1') # Act invalid_key_1 = lambda: None # functions are objects, so this effectively creates an empty object invalid_key_1.get_key_wrap_algorithm = valid_key.get_key_wrap_algorithm invalid_key_1.get_kid = valid_key.get_kid # No attribute wrap_key queue.key_encryption_key = invalid_key_1 with self.assertRaises(AttributeError): queue.send_message(u'message') invalid_key_2 = lambda: None # functions are objects, so this effectively creates an empty object invalid_key_2.wrap_key = valid_key.wrap_key invalid_key_2.get_kid = valid_key.get_kid # No attribute get_key_wrap_algorithm queue.key_encryption_key = invalid_key_2 with self.assertRaises(AttributeError): queue.send_message(u'message') invalid_key_3 = lambda: None # functions are objects, so this effectively creates an empty object invalid_key_3.get_key_wrap_algorithm = valid_key.get_key_wrap_algorithm invalid_key_3.wrap_key = valid_key.wrap_key # No attribute get_kid queue.key_encryption_key = invalid_key_3 with self.assertRaises(AttributeError): queue.send_message(u'message')
def test_message_base64_decode_fails(self, resource_group, location, storage_account, storage_account_key): # Arrange qsc = QueueServiceClient(self._account_url(storage_account.name), storage_account_key) queue = QueueClient( account_url=self._account_url(storage_account.name), queue_name=self.get_resource_name(TEST_QUEUE_PREFIX), credential=storage_account_key, message_encode_policy=None, message_decode_policy=BinaryBase64DecodePolicy()) try: queue.create_queue() except ResourceExistsError: pass message = u'xyz' queue.send_message(message) # Action. with self.assertRaises(DecodeError) as e: queue.peek_messages() # Asserts self.assertNotEqual( -1, str(e.exception).find('Message content is not valid base 64'))
def create_queues(self) -> None: logger.info("creating eventgrid destination queue") name = self.results["deploy"]["func-name"]["value"] key = self.results["deploy"]["func-key"]["value"] account_url = "https://%s.queue.core.windows.net" % name client = QueueServiceClient( account_url=account_url, credential={ "account_name": name, "account_key": key }, ) for queue in [ "file-changes", "task-heartbeat", "node-heartbeat", "proxy", "update-queue", "webhooks", ]: try: client.create_queue(queue) except ResourceExistsError: pass
def test_set_cors(self, resource_group, location, storage_account, storage_account_key): # Arrange qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key) cors_rule1 = CorsRule(['www.xyz.com'], ['GET']) allowed_origins = ['www.xyz.com', "www.ab.com", "www.bc.com"] allowed_methods = ['GET', 'PUT'] max_age_in_seconds = 500 exposed_headers = [ "x-ms-meta-data*", "x-ms-meta-source*", "x-ms-meta-abc", "x-ms-meta-bcd" ] allowed_headers = [ "x-ms-meta-data*", "x-ms-meta-target*", "x-ms-meta-xyz", "x-ms-meta-foo" ] cors_rule2 = CorsRule(allowed_origins, allowed_methods, max_age_in_seconds=max_age_in_seconds, exposed_headers=exposed_headers, allowed_headers=allowed_headers) cors = [cors_rule1, cors_rule2] # Act qsc.set_service_properties(cors=cors) # Assert received_props = qsc.get_service_properties() self._assert_cors_equal(received_props['cors'], cors)
def _get_queues(self) -> List[QueueClient]: return [ QueueServiceClient(q.account_uri).get_queue_client( queue=q.object_name, message_decode_policy=TextBase64DecodePolicy()) for q in self.get_queues_func() ]
def test_sas_add(self, storage_account_name, storage_account_key): # SAS URL is calculated from storage key, so this test runs live only # Arrange qsc = QueueServiceClient( self.account_url(storage_account_name, "queue"), storage_account_key) queue_client = self._get_queue_reference(qsc) queue_client.create_queue() token = generate_queue_sas( queue_client.account_name, queue_client.queue_name, queue_client.credential.account_key, QueueSasPermissions(add=True), datetime.utcnow() + timedelta(hours=1), ) # Act service = QueueClient.from_queue_url( queue_url=queue_client.url, credential=token, ) result = service.send_message(u'addedmessage') # Assert result = next(queue_client.receive_messages()) self.assertEqual(u'addedmessage', result.content)
def ingest_from_blob(self, blob_descriptor: BlobDescriptor, ingestion_properties: IngestionProperties): """ Enqueue an ingest command from azure blobs. To learn more about ingestion methods go to: https://docs.microsoft.com/en-us/azure/data-explorer/ingest-data-overview#ingestion-methods :param azure.kusto.ingest.BlobDescriptor blob_descriptor: An object that contains a description of the blob to be ingested. :param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties. """ try: queues = self._resource_manager.get_ingestion_queues() except KustoServiceError as ex: self._validate_endpoint_service_type() raise ex random_queue = random.choice(queues) queue_service = QueueServiceClient(random_queue.account_uri) authorization_context = self._resource_manager.get_authorization_context( ) ingestion_blob_info = _IngestionBlobInfo( blob_descriptor, ingestion_properties=ingestion_properties, auth_context=authorization_context) ingestion_blob_info_json = ingestion_blob_info.to_json() # TODO: perhaps this needs to be more visible content = ingestion_blob_info_json queue_client = queue_service.get_queue_client( queue=random_queue.object_name, message_encode_policy=TextBase64EncodePolicy()) queue_client.send_message(content=content)
def test_sas_update(self, resource_group, location, storage_account, storage_account_key): # SAS URL is calculated from storage key, so this test runs live only # Arrange qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key) queue_client = self._get_queue_reference(qsc) queue_client.create_queue() queue_client.send_message(u'message1') token = generate_queue_sas( queue_client.account_name, queue_client.queue_name, queue_client.credential.account_key, QueueSasPermissions(update=True), datetime.utcnow() + timedelta(hours=1), ) messages = queue_client.receive_messages() result = next(messages) # Act service = QueueClient.from_queue_url( queue_url=queue_client.url, credential=token, ) service.update_message( result.id, pop_receipt=result.pop_receipt, visibility_timeout=0, content=u'updatedmessage1', ) # Assert result = next(messages) self.assertEqual(u'updatedmessage1', result.content)
def test_user_agent_custom(self, resource_group, location, storage_account, storage_account_key): custom_app = "TestApp/v1.0" service = QueueServiceClient(self._account_url(storage_account.name), credential=storage_account_key, user_agent=custom_app) def callback(response): self.assertTrue('User-Agent' in response.http_request.headers) self.assertEqual( response.http_request.headers['User-Agent'], "TestApp/v1.0 azsdk-python-storage-queue/{} Python/{} ({})". format(VERSION, platform.python_version(), platform.platform())) service.get_service_properties(raw_response_hook=callback) def callback(response): self.assertTrue('User-Agent' in response.http_request.headers) self.assertEqual( response.http_request.headers['User-Agent'], "TestApp/v2.0 azsdk-python-storage-queue/{} Python/{} ({})". format(VERSION, platform.python_version(), platform.platform())) service.get_service_properties(raw_response_hook=callback, user_agent="TestApp/v2.0")
def test_sas_process(self, resource_group, location, storage_account, storage_account_key): # SAS URL is calculated from storage key, so this test runs live only # Arrange qsc = QueueServiceClient(self.account_url(storage_account, "queue"), storage_account_key) queue_client = self._get_queue_reference(qsc) queue_client.create_queue() queue_client.send_message(u'message1') token = generate_queue_sas( queue_client.account_name, queue_client.queue_name, queue_client.credential.account_key, QueueSasPermissions(process=True), datetime.utcnow() + timedelta(hours=1), ) # Act service = QueueClient.from_queue_url( queue_url=queue_client.url, credential=token, ) message = next(service.receive_messages()) # Assert self.assertIsNotNone(message) self.assertNotEqual('', message.id) self.assertEqual(u'message1', message.content)