class FileShare(object):
"""
Information from Azure files service
"""
def __init__(self, account):
self.account_name = account.storage_name()
self.account_key = account.storage_key()
self.files = FileService(
self.account_name, self.account_key
)
def list(self):
"""
list file shares
"""
result = []
try:
for share in self.files.list_shares():
result.append(format(share.name))
except Exception as e:
raise AzureFileShareListError(
'%s: %s' % (type(e).__name__, format(e))
)
return result
def create(self, share_name):
"""
create a file share
"""
try:
self.files.create_share(share_name)
except Exception as e:
raise AzureFileShareCreateError(
'%s: %s' % (type(e).__name__, format(e))
)
def delete(self, share_name):
"""
delete a file share
"""
try:
self.files.delete_share(share_name)
except Exception as e:
raise AzureFileShareDeleteError(
'%s: %s' % (type(e).__name__, format(e))
)
def discover_resources(**kwargs):
discovered_azure_sql = []
for handler in AzureARMHandler.objects.all():
set_progress('Connecting to Azure storage \
files for handler: {}'.format(handler))
credentials = ServicePrincipalCredentials(
client_id=handler.client_id,
secret=handler.secret,
tenant=handler.tenant_id
)
azure_client = storage.StorageManagementClient(
credentials, handler.serviceaccount)
azure_resources_client = resources.ResourceManagementClient(
credentials, handler.serviceaccount)
for resource_group in azure_resources_client.resource_groups.list():
try:
for st in azure_client.storage_accounts.list_by_resource_group(resource_group.name)._get_next().json()['value']:
res = azure_client.storage_accounts.list_keys(
resource_group.name, st['name'])
keys = res.keys
file_service = FileService(
account_name=st['name'], account_key=keys[1].value)
for share in file_service.list_shares():
for file in file_service.list_directories_and_files(share_name=share.name).items:
if type(file) is File:
discovered_azure_sql.append(
{
'name': share.name + '-' + file.name,
'azure_storage_file_name': file.name,
'azure_file_identifier': share.name + '-' + file.name,
'azure_storage_file_share_name': share.name,
'resource_group_name': resource_group.name,
'azure_rh_id': handler.id,
'azure_storage_account_name': st['name'],
'azure_account_key': keys[0].value,
'azure_account_key_fallback': keys[1].value
}
)
except:
continue
return discovered_azure_sql
class StorageHelper(object):
"""Handle details related to a single storage account and share.
Instantiate this object with information sufficient to
uniquely identify a storage account and a file share within it.
Then .account can be used to retrieve the Azure SDK for Python
object corresponding to the account, and .key can be used to
get an access key for it.
For both those properties, if the value mentioned doesn't exist,
it will be created upon first property access.
"""
def __init__(self,
client_data,
resource_helper,
name,
account=None,
default_share='share'):
self.name = name
self.default_share = default_share
self._account = account
self._key = os.environ.get('AZURE_STORAGE_KEY')
self.resource_helper = resource_helper
self.client = StorageManagementClient(*client_data)
self.file_service = FileService(
account_name=self.account.name,
account_key=self.key,
)
@property
def account(self):
"""Return the managed StorageAccounts object.
If no such account exists, create it first.
"""
if self._account is None:
print('Creating storage account...')
# Error to create storage account if it already exists!
name_check = self.client.storage_accounts.check_name_availability(
self.name)
if name_check.name_available:
storage_creation = self.client.storage_accounts.create(
self.resource_helper.group.name, self.name,
StorageAccountCreateParameters(
sku=StorageAccountSku(StorageSkuName.standard_lrs),
kind=StorageKind.storage,
location=self.resource_helper.group.location,
))
storage = storage_creation.result()
else:
try:
storage = self.client.storage_accounts.get_properties(
self.resource_helper.group.name, self.name)
except CloudError:
print('Storage account {} already exists'
' in a resource group other than {}.'.format(
self.name, self.resource_helper.group.name))
print('Got storage account:', storage.name)
self._account = storage
return self._account
@property
def key(self):
"""Get the first available storage key.
This will crash if there are no available storage keys,
which is unlikely since two are created along with a storage account.
"""
if self._key is None:
storage_keys = self.client.storage_accounts.list_keys(
self.resource_helper.group.name, self.account.name)
self._key = next(iter(storage_keys.keys)).value
return self._key
def upload_file(self, path, sharename):
"""Upload a file into the default share on the storage account.
If the share doesn't exist, create it first.
"""
self.file_service.create_file_from_path(
self.default_share if sharename is None else sharename,
None,
os.path.basename(path),
path,
)
return '/'.join([self.default_share, os.path.basename(path)])
def download_file(self, sharename, filename):
file_service.get_file_to_path(sharename, None, filename, filename)
def delete_file(self, sharename, filename):
file_service.delete_file(sharename, None, filename)
def create_share(self, sharename):
self.file_service.create_share(sharename)
def create_directory(self, sharename, directoryname):
self.file_service.create_directory(sharename, directoryname)
def list_directories_and_files(self, sharename):
generator = self.file_service.list_directories_and_files(sharename)
return [file_or_dir.name for file_or_dir in generator]
def list_shares(self):
shares = list(self.file_service.list_shares(include_snapshots=True))
sharelist = [fileshare.name for fileshare in shares]
print(sharelist)
return sharelist
class AzureHandler(Handler):
class Settings(BaseSettings):
client_id: UUID
client_secret: SecretStr
keyvault_url: HttpUrl
storage_blob_url: HttpUrl
tenant_id: UUID
storage_name: str
storage_share_name: str
storage_sas: SecretStr
def __init__(self, env_file) -> None:
self.settings = AzureHandler.Settings(_env_file=env_file)
self.credential = ClientSecretCredential(
str(self.settings.tenant_id),
str(self.settings.client_id),
self.settings.client_secret.get_secret_value(),
)
# Create File and KeyVault Secret Clients
self.secret_index = {}
self.secretsclient = SecretClient(self.settings.keyvault_url,
self.credential)
self.fileclient = FileService(
self.settings.storage_name,
sas_token=self.settings.storage_sas.get_secret_value(),
)
try:
self.fileclient.create_share("lootmarshal")
self.fileclient.create_directory("lootmarshal", "binary_dumps")
asyncio.create_task(self.build_secret_index())
except Exception as e:
raise e
async def validate(self) -> bool:
try:
if not len(self.fileclient.list_shares(num_results=1)):
raise Exception(
"Azure FileService Client could not list shares! Verify authentication."
)
[p async for p in self.secretsclient.list_properties_of_secrets()]
return True
except Exception as e:
raise e
async def build_secret_index(self):
while True:
try:
self.secret_index = [{
"name": p.name,
"content_type": p.content_type,
"tags": p.tags
} async for p in self.secretsclient.list_properties_of_secrets(
)]
except Exception as e:
raise e
await asyncio.sleep(10)
def write_file(self, directory: str, name: str, file: bytes):
name = f"{name}_{time.strftime('%Y%m%d-%H%M%S')}"
logging.info(
f"Writing {name} ({len(file)} bytes) to {directory}/{name}.")
self.fileclient.create_file_from_bytes(
self.settings.storage_share_name, directory, name, file)
async def read_secret(self, name) -> dict:
try:
secret = await self.secretsclient.get_secret(name)
formatted = {
"name": secret.name,
"value": secret.value,
"content_type": secret.properties.content_type,
"tags": secret.properties.tags,
}
return formatted
except Exception as e:
raise HTTPException(status_code=404, detail=str(e))
async def write_secret(self, name, value, content_type, tags):
try:
return await self.secretsclient.set_secret(
name, value, content_type=content_type, tags=tags)
except Exception as e:
raise HTTPException(status_code=404, detail=str(e))
async def list_secrets(self):
try:
secret_list = []
async for props in self.secretsclient.list_properties_of_secrets():
secret = await self.secretsclient.get_secret(props.name)
formatted = {
"name": secret.name,
"value": secret.value,
"content_type": secret.properties.content_type,
"tags": secret.properties.tags,
}
secret_list.append(formatted)
return json.dumps(secret_list, indent=4)
except Exception as e:
raise HTTPException(status_code=404, detail=str(e))
