class FileShare(object): """ Information from Azure files service """ def __init__(self, account): self.account_name = account.storage_name() self.account_key = account.storage_key() self.files = FileService( self.account_name, self.account_key ) def list(self): """ list file shares """ result = [] try: for share in self.files.list_shares(): result.append(format(share.name)) except Exception as e: raise AzureFileShareListError( '%s: %s' % (type(e).__name__, format(e)) ) return result def create(self, share_name): """ create a file share """ try: self.files.create_share(share_name) except Exception as e: raise AzureFileShareCreateError( '%s: %s' % (type(e).__name__, format(e)) ) def delete(self, share_name): """ delete a file share """ try: self.files.delete_share(share_name) except Exception as e: raise AzureFileShareDeleteError( '%s: %s' % (type(e).__name__, format(e)) )
def discover_resources(**kwargs): discovered_azure_sql = [] for handler in AzureARMHandler.objects.all(): set_progress('Connecting to Azure storage \ files for handler: {}'.format(handler)) credentials = ServicePrincipalCredentials( client_id=handler.client_id, secret=handler.secret, tenant=handler.tenant_id ) azure_client = storage.StorageManagementClient( credentials, handler.serviceaccount) azure_resources_client = resources.ResourceManagementClient( credentials, handler.serviceaccount) for resource_group in azure_resources_client.resource_groups.list(): try: for st in azure_client.storage_accounts.list_by_resource_group(resource_group.name)._get_next().json()['value']: res = azure_client.storage_accounts.list_keys( resource_group.name, st['name']) keys = res.keys file_service = FileService( account_name=st['name'], account_key=keys[1].value) for share in file_service.list_shares(): for file in file_service.list_directories_and_files(share_name=share.name).items: if type(file) is File: discovered_azure_sql.append( { 'name': share.name + '-' + file.name, 'azure_storage_file_name': file.name, 'azure_file_identifier': share.name + '-' + file.name, 'azure_storage_file_share_name': share.name, 'resource_group_name': resource_group.name, 'azure_rh_id': handler.id, 'azure_storage_account_name': st['name'], 'azure_account_key': keys[0].value, 'azure_account_key_fallback': keys[1].value } ) except: continue return discovered_azure_sql
class StorageHelper(object): """Handle details related to a single storage account and share. Instantiate this object with information sufficient to uniquely identify a storage account and a file share within it. Then .account can be used to retrieve the Azure SDK for Python object corresponding to the account, and .key can be used to get an access key for it. For both those properties, if the value mentioned doesn't exist, it will be created upon first property access. """ def __init__(self, client_data, resource_helper, name, account=None, default_share='share'): self.name = name self.default_share = default_share self._account = account self._key = os.environ.get('AZURE_STORAGE_KEY') self.resource_helper = resource_helper self.client = StorageManagementClient(*client_data) self.file_service = FileService( account_name=self.account.name, account_key=self.key, ) @property def account(self): """Return the managed StorageAccounts object. If no such account exists, create it first. """ if self._account is None: print('Creating storage account...') # Error to create storage account if it already exists! name_check = self.client.storage_accounts.check_name_availability( self.name) if name_check.name_available: storage_creation = self.client.storage_accounts.create( self.resource_helper.group.name, self.name, StorageAccountCreateParameters( sku=StorageAccountSku(StorageSkuName.standard_lrs), kind=StorageKind.storage, location=self.resource_helper.group.location, )) storage = storage_creation.result() else: try: storage = self.client.storage_accounts.get_properties( self.resource_helper.group.name, self.name) except CloudError: print('Storage account {} already exists' ' in a resource group other than {}.'.format( self.name, self.resource_helper.group.name)) print('Got storage account:', storage.name) self._account = storage return self._account @property def key(self): """Get the first available storage key. This will crash if there are no available storage keys, which is unlikely since two are created along with a storage account. """ if self._key is None: storage_keys = self.client.storage_accounts.list_keys( self.resource_helper.group.name, self.account.name) self._key = next(iter(storage_keys.keys)).value return self._key def upload_file(self, path, sharename): """Upload a file into the default share on the storage account. If the share doesn't exist, create it first. """ self.file_service.create_file_from_path( self.default_share if sharename is None else sharename, None, os.path.basename(path), path, ) return '/'.join([self.default_share, os.path.basename(path)]) def download_file(self, sharename, filename): file_service.get_file_to_path(sharename, None, filename, filename) def delete_file(self, sharename, filename): file_service.delete_file(sharename, None, filename) def create_share(self, sharename): self.file_service.create_share(sharename) def create_directory(self, sharename, directoryname): self.file_service.create_directory(sharename, directoryname) def list_directories_and_files(self, sharename): generator = self.file_service.list_directories_and_files(sharename) return [file_or_dir.name for file_or_dir in generator] def list_shares(self): shares = list(self.file_service.list_shares(include_snapshots=True)) sharelist = [fileshare.name for fileshare in shares] print(sharelist) return sharelist
class AzureHandler(Handler): class Settings(BaseSettings): client_id: UUID client_secret: SecretStr keyvault_url: HttpUrl storage_blob_url: HttpUrl tenant_id: UUID storage_name: str storage_share_name: str storage_sas: SecretStr def __init__(self, env_file) -> None: self.settings = AzureHandler.Settings(_env_file=env_file) self.credential = ClientSecretCredential( str(self.settings.tenant_id), str(self.settings.client_id), self.settings.client_secret.get_secret_value(), ) # Create File and KeyVault Secret Clients self.secret_index = {} self.secretsclient = SecretClient(self.settings.keyvault_url, self.credential) self.fileclient = FileService( self.settings.storage_name, sas_token=self.settings.storage_sas.get_secret_value(), ) try: self.fileclient.create_share("lootmarshal") self.fileclient.create_directory("lootmarshal", "binary_dumps") asyncio.create_task(self.build_secret_index()) except Exception as e: raise e async def validate(self) -> bool: try: if not len(self.fileclient.list_shares(num_results=1)): raise Exception( "Azure FileService Client could not list shares! Verify authentication." ) [p async for p in self.secretsclient.list_properties_of_secrets()] return True except Exception as e: raise e async def build_secret_index(self): while True: try: self.secret_index = [{ "name": p.name, "content_type": p.content_type, "tags": p.tags } async for p in self.secretsclient.list_properties_of_secrets( )] except Exception as e: raise e await asyncio.sleep(10) def write_file(self, directory: str, name: str, file: bytes): name = f"{name}_{time.strftime('%Y%m%d-%H%M%S')}" logging.info( f"Writing {name} ({len(file)} bytes) to {directory}/{name}.") self.fileclient.create_file_from_bytes( self.settings.storage_share_name, directory, name, file) async def read_secret(self, name) -> dict: try: secret = await self.secretsclient.get_secret(name) formatted = { "name": secret.name, "value": secret.value, "content_type": secret.properties.content_type, "tags": secret.properties.tags, } return formatted except Exception as e: raise HTTPException(status_code=404, detail=str(e)) async def write_secret(self, name, value, content_type, tags): try: return await self.secretsclient.set_secret( name, value, content_type=content_type, tags=tags) except Exception as e: raise HTTPException(status_code=404, detail=str(e)) async def list_secrets(self): try: secret_list = [] async for props in self.secretsclient.list_properties_of_secrets(): secret = await self.secretsclient.get_secret(props.name) formatted = { "name": secret.name, "value": secret.value, "content_type": secret.properties.content_type, "tags": secret.properties.tags, } secret_list.append(formatted) return json.dumps(secret_list, indent=4) except Exception as e: raise HTTPException(status_code=404, detail=str(e))