def get_storage_account_details(creds, resource_group_name, account_name): storage_client = StorageManagementClient(creds) account_result = storage_client.storage_accounts.get_properties(resource_group_name, account_name) keys_result = storage_client.storage_accounts.list_keys(resource_group_name, account_name) account_key = keys_result.storage_account_keys.key1 account = CloudStorageAccount(account_name, account_key) blob_service = account.create_blob_service() file_service = account.create_file_service() queue_service = account.create_queue_service() table_service = account.create_table_service() model = StorageAccountDetails() model.account_props = account_result.storage_account model.account_keys = keys_result.storage_account_keys model.blob_containers = blob_service.iterate_containers() model.queues = queue_service.iterate_queues() # TODO: find out why listing shares doesn't work # model.shares = file_service.iterate_shares() model.shares = [] model.tables = table_service.iterate_tables() model.blob_service_properties = blob_service.get_blob_service_properties() model.queue_service_properties = queue_service.get_queue_service_properties() model.table_service_properties = table_service.get_table_service_properties() return model
def get_storage_account_details(subscription_id, creds, resource_group_name, account_name): storage_client = StorageManagementClient(creds, subscription_id) account_result = storage_client.storage_accounts.get_properties( resource_group_name, account_name, ) storage_account_keys = storage_client.storage_accounts.list_keys( resource_group_name, account_name, ) account_key = storage_account_keys.key1 account = CloudStorageAccount(account_name, account_key) blob_service = account.create_blob_service() file_service = account.create_file_service() queue_service = account.create_queue_service() table_service = account.create_table_service() model = StorageAccountDetails() model.account_props = account_result model.account_keys = storage_account_keys model.blob_containers = blob_service.iterate_containers() model.queues = queue_service.iterate_queues() #TODO: find out why listing shares doesn't work #model.shares = file_service.iterate_shares() model.shares = [] model.tables = table_service.iterate_tables() model.blob_service_properties = blob_service.get_blob_service_properties() model.queue_service_properties = queue_service.get_queue_service_properties( ) model.table_service_properties = table_service.get_table_service_properties( ) return model
class CloudStorageAccountTest(AzureTestCase): def setUp(self): self.account = CloudStorageAccount( account_name=credentials.getStorageServicesName(), account_key=credentials.getStorageServicesKey()) #--Test cases -------------------------------------------------------- def test_create_blob_service(self): # Arrange # Act service = self.account.create_blob_service() # Assert self.assertIsNotNone(service) self.assertIsInstance(service, BlobService) self.assertEqual(service.account_name, credentials.getStorageServicesName()) self.assertEqual(service.account_key, credentials.getStorageServicesKey()) def test_create_blob_service_empty_credentials(self): # Arrange # Act bad_account = CloudStorageAccount('', '') with self.assertRaises(WindowsAzureError): service = bad_account.create_blob_service() # Assert def test_create_table_service(self): # Arrange # Act service = self.account.create_table_service() # Assert self.assertIsNotNone(service) self.assertIsInstance(service, TableService) self.assertEqual(service.account_name, credentials.getStorageServicesName()) self.assertEqual(service.account_key, credentials.getStorageServicesKey()) def test_create_queue_service(self): # Arrange # Act service = self.account.create_queue_service() # Assert self.assertIsNotNone(service) self.assertIsInstance(service, QueueService) self.assertEqual(service.account_name, credentials.getStorageServicesName()) self.assertEqual(service.account_key, credentials.getStorageServicesKey())
class StorageAccountTest(ExtendedTestCase): def setUp(self): self.account_name = 'storagename' self.account_key = 'NzhL3hKZbJBuJ2484dPTR+xF30kYaWSSCbs2BzLgVVI1woqeST/1IgqaLm6QAOTxtGvxctSNbIR/1hW8yH+bJg==' self.account = CloudStorageAccount(self.account_name, self.account_key) #--Test cases -------------------------------------------------------- def test_create_blob_service(self): # Arrange # Act service = self.account.create_blob_service() # Assert self.assertIsNotNone(service) self.assertIsInstance(service, BlobService) self.assertEqual(service.account_name, self.account_name) self.assertEqual(service.account_key, self.account_key) def test_create_blob_service_empty_credentials(self): # Arrange # Act bad_account = CloudStorageAccount('', '') with self.assertRaises(ValueError): service = bad_account.create_blob_service() # Assert def test_create_table_service(self): # Arrange # Act service = self.account.create_table_service() # Assert self.assertIsNotNone(service) self.assertIsInstance(service, TableService) self.assertEqual(service.account_name, self.account_name) self.assertEqual(service.account_key, self.account_key) def test_create_queue_service(self): # Arrange # Act service = self.account.create_queue_service() # Assert self.assertIsNotNone(service) self.assertIsInstance(service, QueueService) self.assertEqual(service.account_name, self.account_name) self.assertEqual(service.account_key, self.account_key)
class StorageAccountTest(StorageTestCase): def setUp(self): self.account_name = 'storagename' self.account_key = 'NzhL3hKZbJBuJ2484dPTR+xF30kYaWSSCbs2BzLgVVI1woqeST/1IgqaLm6QAOTxtGvxctSNbIR/1hW8yH+bJg==' self.account = CloudStorageAccount(self.account_name, self.account_key) #--Test cases -------------------------------------------------------- def test_create_blob_service(self): # Arrange # Act service = self.account.create_blob_service() # Assert self.assertIsNotNone(service) self.assertIsInstance(service, BlobService) self.assertEqual(service.account_name, self.account_name) self.assertEqual(service.account_key, self.account_key) def test_create_blob_service_empty_credentials(self): # Arrange # Act bad_account = CloudStorageAccount('', '') with self.assertRaises(ValueError): service = bad_account.create_blob_service() # Assert def test_create_table_service(self): # Arrange # Act service = self.account.create_table_service() # Assert self.assertIsNotNone(service) self.assertIsInstance(service, TableService) self.assertEqual(service.account_name, self.account_name) self.assertEqual(service.account_key, self.account_key) def test_create_queue_service(self): # Arrange # Act service = self.account.create_queue_service() # Assert self.assertIsNotNone(service) self.assertIsInstance(service, QueueService) self.assertEqual(service.account_name, self.account_name) self.assertEqual(service.account_key, self.account_key)
def submit_video_encoding_job(request, video_guid=None): blob_name = format_blob_name(user=request.user, video_guid=video_guid) csa = CloudStorageAccount( account_name=settings.AZURE_CONFIG["account_name"], account_key=settings.AZURE_CONFIG["account_key"] ) qs = csa.create_queue_service() qs.put_message(queue_name=settings.AZURE_CONFIG["video_encode_queue"], message_text=base64.b64encode(blob_name)) return None
def main(req: func.HttpRequest) -> func.HttpResponse: logging.info('Python HTTP Move trigger received a request') logging.debug('Creating queue service') account = CloudStorageAccount( account_name=os.getenv('AZURE_STORAGE_ACCOUNT'), account_key=os.getenv('AZURE_STORAGE_ACCESS_KEY')) headers_dict = { "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Origin": "*", "Access-Control-Allow-Methods": "Get" } queue_service = account.create_queue_service() guid = req.params.get('guid') if not guid: error = f'Check Url paramaters. No guid variable was found' return func.HttpResponse(error, headers=headers_dict, status_code=400) guid = uuid.UUID(guid) logging.info("Creating Dicts") await_queue_name = os.getenv('AZURE_AWAIT_QUEUE_NAME') done_queue_name = os.getenv('AZURE_DONE_QUEUE_NAME') logging.info("Searching in await-processing queue") messages_awaiting = queue_service.get_messages(await_queue_name, num_messages=32) messages_info_by_guid = {} await_schema = move_schema.AwaitSchema() found = False for m in messages_awaiting: message_guid = await_schema.loads(decode_message(m))['guid'] messages_info_by_guid[message_guid] = (m.id, m.pop_receipt) if message_guid == guid: found = True break if found: [m_id, m_pop_receipt] = messages_info_by_guid[guid] queue_service.delete_message(await_queue_name, m_id, m_pop_receipt) done_schema = move_schema.DoneSchema() done_dict = {} done_dict['guid'] = guid done_dict['error'] = random.sample([None, "random error"], 1)[0] message = done_schema.dumps(done_dict) queue_service.put_message(queue_name=done_queue_name, content=encode_message(message), time_to_live=604800) return func.HttpResponse(f'Ok', headers=headers_dict) else: return func.HttpResponse(f'Not found', headers=headers_dict, status_code=400)
def submit_video_encoding_job(request, video_guid = None): blob_name = format_blob_name(user = request.user, video_guid = video_guid) csa = CloudStorageAccount( account_name = settings.AZURE_CONFIG['account_name'], account_key = settings.AZURE_CONFIG['account_key']) qs = csa.create_queue_service() qs.put_message( queue_name = settings.AZURE_CONFIG['video_encode_queue'], message_text = base64.b64encode(blob_name)) return None
def _turn_on_minute_metrics(self, name, key, metric_table_retention_period): account = CloudStorageAccount(account_name=name, account_key=key, sas_token=None) retention_policy = RetentionPolicy(enabled=True, days=metric_table_retention_period) metrics = Metrics(enabled=True, include_apis=True, retention_policy=retention_policy) table_service = account.create_table_service() table_service.set_table_service_properties(minute_metrics=metrics) blob_service = account.create_page_blob_service() blob_service.set_blob_service_properties(minute_metrics=metrics) file_service = account.create_file_service() file_service.set_file_service_properties(minute_metrics=metrics) queue_service = account.create_queue_service() queue_service.set_queue_service_properties(minute_metrics=metrics)
# blob services. # # See http://go.microsoft.com/fwlink/?linkid=246933 for Storage documentation. # STORAGE_ACCOUNT_NAME = '__paste_your_storage_account_name_here__' STORAGE_ACCOUNT_KEY = '__paste_your_storage_key_here__' if os.environ.get('EMULATED', '').lower() == 'true': # Running in the emulator, so use the development storage account storage_account = CloudStorageAccount(None, None) else: storage_account = CloudStorageAccount(STORAGE_ACCOUNT_NAME, STORAGE_ACCOUNT_KEY) blob_service = storage_account.create_blob_service() table_service = storage_account.create_table_service() queue_service = storage_account.create_queue_service() # # Service Bus is a messaging solution for applications. It sits between # components of your applications and enables them to exchange messages in a # loosely coupled way for improved scale and resiliency. # # See http://go.microsoft.com/fwlink/?linkid=246934 for Service Bus documentation. # SERVICE_BUS_NAMESPACE = '__paste_your_service_bus_namespace_here__' SERVICE_BUS_KEY = '__paste_your_service_bus_key_here__' bus_service = ServiceBusService(SERVICE_BUS_NAMESPACE, SERVICE_BUS_KEY, issuer='owner') if __name__ == '__main__': while True:
# # See http://go.microsoft.com/fwlink/?linkid=246933 for Storage documentation. # STORAGE_ACCOUNT_NAME = '__paste_your_storage_account_name_here__' STORAGE_ACCOUNT_KEY = '__paste_your_storage_key_here__' if os.environ.get('EMULATED', '').lower() == 'true': # Running in the emulator, so use the development storage account storage_account = CloudStorageAccount(None, None) else: storage_account = CloudStorageAccount(STORAGE_ACCOUNT_NAME, STORAGE_ACCOUNT_KEY) blob_service = storage_account.create_blob_service() table_service = storage_account.create_table_service() queue_service = storage_account.create_queue_service() # # Service Bus is a messaging solution for applications. It sits between # components of your applications and enables them to exchange messages in a # loosely coupled way for improved scale and resiliency. # # See http://go.microsoft.com/fwlink/?linkid=246934 for Service Bus documentation. # SERVICE_BUS_NAMESPACE = '__paste_your_service_bus_namespace_here__' SERVICE_BUS_KEY = '__paste_your_service_bus_key_here__' bus_service = ServiceBusService(SERVICE_BUS_NAMESPACE, SERVICE_BUS_KEY, issuer='owner') if __name__ == '__main__':
class StorageAccountTest(StorageTestCase): def setUp(self): super(StorageAccountTest, self).setUp() self.account_name = self.settings.STORAGE_ACCOUNT_NAME self.account_key = self.settings.STORAGE_ACCOUNT_KEY self.sas_token = '?sv=2015-04-05&st=2015-04-29T22%3A18%3A26Z&se=2015-04-30T02%3A23%3A26Z&sr=b&sp=rw&sip=168.1.5.60-168.1.5.70&spr=https&sig=Z%2FRHIX5Xcg0Mq2rqI3OlWTjEg2tYkboXr1P9ZUXDtkk%3D' self.account = CloudStorageAccount(self.account_name, self.account_key) #--Helpers----------------------------------------------------------------- def validate_service(self, service, type): self.assertIsNotNone(service) self.assertIsInstance(service, type) self.assertEqual(service.account_name, self.account_name) self.assertEqual(service.account_key, self.account_key) #--Test cases -------------------------------------------------------- def test_create_block_blob_service(self): # Arrange # Act service = self.account.create_block_blob_service() # Assert self.validate_service(service, BlockBlobService) def test_create_page_blob_service(self): # Arrange # Act service = self.account.create_page_blob_service() # Assert self.validate_service(service, PageBlobService) def test_create_append_blob_service(self): # Arrange # Act service = self.account.create_append_blob_service() # Assert self.validate_service(service, AppendBlobService) def test_create_table_service(self): # Arrange # Act service = self.account.create_table_service() # Assert self.validate_service(service, TableService) def test_create_queue_service(self): # Arrange # Act service = self.account.create_queue_service() # Assert self.validate_service(service, QueueService) def test_create_file_service(self): # Arrange # Act service = self.account.create_file_service() # Assert self.validate_service(service, FileService) def test_create_service_no_key(self): # Arrange # Act bad_account = CloudStorageAccount('', '') with self.assertRaises(ValueError): service = bad_account.create_block_blob_service() # Assert def test_create_account_sas(self): # Arrange # Act sas_account = CloudStorageAccount(self.account_name, sas_token=self.sas_token) service = sas_account.create_block_blob_service() # Assert self.assertIsNotNone(service) self.assertEqual(service.account_name, self.account_name) self.assertIsNone(service.account_key) self.assertEqual(service.sas_token, self.sas_token) def test_create_account_sas_and_key(self): # Arrange # Act account = CloudStorageAccount(self.account_name, self.account_key, self.sas_token) service = account.create_block_blob_service() # Assert self.validate_service(service, BlockBlobService) def test_create_account_emulated(self): # Arrange # Act account = CloudStorageAccount(is_emulated=True) service = account.create_block_blob_service() # Assert self.assertIsNotNone(service) self.assertEqual(service.account_name, 'devstoreaccount1') self.assertIsNotNone(service.account_key) @record def test_generate_account_sas(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recordingfile(self.test_mode): return # Arrange token = self.account.generate_shared_access_signature( Services.BLOB, ResourceTypes.OBJECT, AccountPermissions.READ, datetime.utcnow() + timedelta(hours=1), ) service = self.account.create_block_blob_service() data = b'shared access signature with read permission on blob' container_name='container1' blob_name = 'blob1.txt' try: service.create_container(container_name) service.create_blob_from_bytes(container_name, blob_name, data) # Act url = service.make_blob_url( container_name, blob_name, sas_token=token, ) response = requests.get(url) # Assert self.assertTrue(response.ok) self.assertEqual(data, response.content) finally: service.delete_container(container_name)
class StorageQueueContext(): """Initializes the repository with the specified settings dict. Required settings in config dict are: - AZURE_STORAGE_NAME - AZURE_STORAGE_KEY - AZURE_REQUIRE_ENCRYPTION - AZURE_KEY_IDENTIFIER - AZURE_SECRET_KEY - AZURE_STORAGE_IS_EMULATED """ _account = None _account_name = '' _account_key = '' _is_emulated = False _modeldefinitions = [] REGISTERED = True """ decorators """ def get_modeldefinition(registered=False): def wrap(func): @wraps(func) def wrapper(self, storagemodel, modeldefinition=None, *args, **kwargs): """ modeldefinition already determined """ if not modeldefinition is None: return func(self, storageobject, modeldefinition, *args, **kwargs) """ find modeldefinition for StorageQueueModel or StorageQueueModel """ if isinstance(storagemodel, StorageQueueModel): definitionlist = [definition for definition in self._modeldefinitions if definition['modelname'] == storagemodel.__class__.__name__] else: log.info('Argument is not an StorageQueueModel') raise Exception("Argument is not an StorageQueueModel") if len(definitionlist) == 1: modeldefinition = definitionlist[0] elif len(definitionlist) > 1: raise Exception("multiple registration for model") if registered and (not isinstance(modeldefinition, dict)): raise Exception("Please register Model first") return func(self, storagemodel, modeldefinition, *args, **kwargs) return wrapper return wrap def __init__(self, **kwargs): """ parse kwargs """ self._account_name = kwargs.get('AZURE_STORAGE_NAME', '') self._account_key = kwargs.get('AZURE_STORAGE_KEY', '') self._is_emulated = kwargs.get('AZURE_STORAGE_IS_EMULATED', False) self._key_identifier = kwargs.get('AZURE_KEY_IDENTIFIER', '') self._secret_key = kwargs.get('AZURE_SECRET_KEY', '') """ account & service init """ if self._is_emulated: self._account = CloudStorageAccount(is_emulated=True) elif self._account_name != '' and self._account_key != '': self._account = CloudStorageAccount(self._account_name, self._account_key) else: raise AzureException """ registered models """ self._modeldefinitions = [] def __create__(self, modeldefinition:dict) -> bool: if (not modeldefinition['queueservice'] is None): try: modeldefinition['queueservice'].create_queue(modeldefinition['queuename']) return True except Exception as e: msg = 'failed to create {} with error {}'.format(modeldefinition['queuename'], e) raise AzureStorageWrapException(msg=msg) else: return True pass def __deletequeue__(self, modeldefinition:dict) -> bool: if (not modeldefinition['queueservice'] is None): try: modeldefinition['queueservice'].delete_queue(modeldefinition['queuename']) return True except Exception as e: msg = 'failed to delete {} with error {}'.format(modeldefinition['queuename'], e) raise AzureStorageWrapException(msg=msg) else: return True pass @get_modeldefinition() def register_model(self, storagemodel:object, modeldefinition = None): """ set up an Queueservice for an StorageQueueModel in your Azure Storage Account Will create the Queue if not exist! required Parameter is: - storagemodel: StorageQueueModel(Object) """ if modeldefinition is None: """ test if queuename already exists """ if [model for model in self._modeldefinitions if model['queuename'] == storagemodel._queuename]: raise NameConventionError(storagemodel._queuename) """ test if queuename fits to azure naming rules """ if not test_azurestorage_nameconventions(storagemodel._queuename, 'StorageQueueModel'): raise NameConventionError(storagemodel._queuename) """ now register model """ modeldefinition = { 'modelname': storagemodel.__class__.__name__, 'queuename': storagemodel._queuename, 'encrypt': storagemodel._encrypt, 'queueservice': self._account.create_queue_service() } """ encrypt queue service """ if modeldefinition['encrypt']: # Create the KEK used for encryption. # KeyWrapper is the provided sample implementation, but the user may use their own object as long as it implements the interface above. kek = KeyWrapper(self._key_identifier, self._secret_key) # Key identifier # Create the key resolver used for decryption. # KeyResolver is the provided sample implementation, but the user may use whatever implementation they choose so long as the function set on the service object behaves appropriately. key_resolver = KeyResolver() key_resolver.put_key(kek) # Set the require Encryption, KEK and key resolver on the service object. modeldefinition['queueservice'].require_encryption = True modeldefinition['queueservice'].key_encryption_key = kek modeldefinition['queueservice'].key_resolver_funcion = key_resolver.resolve_key self.__create__(modeldefinition) self._modeldefinitions.append(modeldefinition) log.info('model {} registered successfully. Models are {!s}.'.format(modeldefinition['modelname'], [model['modelname'] for model in self._modeldefinitions])) else: log.info('model {} already registered. Models are {!s}.'.format(modeldefinition['modelname'], [model['modelname'] for model in self._modeldefinitions])) pass @get_modeldefinition(REGISTERED) def unregister_model(self, storagemodel:object, modeldefinition = None, delete_queue=False): """ clear up an Queueservice for an StorageQueueModel in your Azure Storage Account Will delete the hole Queue if delete_queue Flag is True! required Parameter is: - storagemodel: StorageQueueModel(Object) Optional Parameter is: - delete_queue: bool """ """ remove from modeldefinitions """ for i in range(len(self._modeldefinitions)): if self._modeldefinitions[i]['modelname'] == modeldefinition['modelname']: del self._modeldefinitions[i] break """ delete queue from storage if delete_queue == True """ if delete_queue: self.__deletequeue__(modeldefinition) log.info('model {} unregistered successfully. Models are {!s}'.format(modeldefinition['modelname'], [model['modelname'] for model in self._modeldefinitions])) pass @get_modeldefinition(REGISTERED) def put(self, storagemodel:object, modeldefinition = None) -> StorageQueueModel: """ insert queue message into storage """ try: message = modeldefinition['queueservice'].put_message(storagemodel._queuename, storagemodel.getmessage()) storagemodel.mergemessage(message) except Exception as e: storagemodel = None msg = 'can not save queue message: queue {} with message {} because {!s}'.format(storagemodel._queuename, storagemodel.content, e) raise AzureStorageWrapException(msg=msg) finally: return storagemodel @get_modeldefinition(REGISTERED) def peek(self, storagemodel:object, modeldefinition = None) -> StorageQueueModel: """ lookup the next message in queue """ try: messages = modeldefinition['queueservice'].peek_messages(storagemodel._queuename, num_messages=1) """ parse retrieved message """ for message in messages: storagemodel.mergemessage(message) """ no message retrieved ?""" if storagemodel.id is None: storagemodel = None except Exception as e: storagemodel = None msg = 'can not peek queue message: queue {} with message {} because {!s}'.format(storagemodel._queuename, storagemodel.content, e) raise AzureStorageWrapException(msg=msg) finally: return storagemodel @get_modeldefinition(REGISTERED) def get(self, storagemodel:object, modeldefinition = None, hide = 0) -> StorageQueueModel: """ get the next message in queue """ try: if hide > 0: messages = modeldefinition['queueservice'].get_messages(storagemodel._queuename, num_messages=1, visibility_timeout = hide) else: messages = modeldefinition['queueservice'].get_messages(storagemodel._queuename, num_messages=1) """ parse retrieved message """ for message in messages: storagemodel.mergemessage(message) """ no message retrieved ?""" if storagemodel.id is None: storagemodel = None except Exception as e: storagemodel = None msg = 'can not peek queue message: queue {} with message {} because {!s}'.format(storagemodel._queuename, storagemodel.content, e) raise AzureStorageWrapException(msg=msg) finally: return storagemodel @get_modeldefinition(REGISTERED) def update(self, storagemodel:object, modeldefinition = None, hide = 0) -> StorageQueueModel: """ update the message in queue """ if (storagemodel.id != '') and (storagemodel.pop_receipt != '') and (not storagemodel.id is None) and (not storagemodel.pop_receipt is None): try: content = storagemodel.getmessage() message = modeldefinition['queueservice'].update_message(storagemodel._queuename, storagemodel.id, storagemodel.pop_receipt, visibility_timeout = hide, content=content) storagemodel.content = content storagemodel.pop_receipt = message.pop_receipt except Exception as e: msg = 'can not update queue message: queue {} with message.id {!s} because {!s}'.format(storagemodel._queuename, storagemodel.id, e) storagemodel = None raise AzureStorageWrapException(msg=msg) else: msg = 'cant update queuemessage in {!s} due to missing id {!s} and/or pop_receipt {!s}'.format(storagemodel._queuename, storagemodel.id, storagemodel.pop_receipt) storagemodel = None raise AzureStorageWrapException(msg=msg) return storagemodel @get_modeldefinition(REGISTERED) def delete(self, storagemodel:object, modeldefinition = None) -> bool: """ delete the message in queue """ deleted = False if (storagemodel.id != '') and (storagemodel.pop_receipt != '') and (not storagemodel.id is None) and (not storagemodel.pop_receipt is None): try: modeldefinition['queueservice'].delete_message(storagemodel._queuename, storagemodel.id, storagemodel.pop_receipt) deleted = True except Exception as e: msg = 'can not delete queue message: queue {} with message.id {!s} because {!s}'.format(storagemodel._queuename, storagemodel.id, e) raise AzureStorageWrapException(msg=msg) else: log.info('cant update queuemessage {} due to missing id and pop_receipt'.format(storagemodel._queuename)) return deleted