def post_file(share_name, path_to_file):
try:
account_name, account_key = get_auth(request.authorization)
file_service = FileService(account_name=account_name,
account_key=account_key)
start_timedelta, expiry_timedelta = get_sas_params(request.args)
directory_name, file_name = get_location(path_to_file)
if request.headers.get('Transfer-Encoding') == 'chunked':
file_service.create_file_from_stream(share_name,
directory_name,
file_name,
request.stream,
count=4096)
else:
file_service.create_file_from_bytes(share_name, directory_name,
file_name, request.get_data())
sas_token = file_service.generate_file_shared_access_signature(
share_name,
directory_name=directory_name,
file_name=file_name,
permission=SharePermissions(read=True),
expiry=datetime.now() + expiry_timedelta,
start=start_timedelta,
id=None,
ip=None,
protocol='https',
cache_control=request.headers.get('Cache-Control'),
content_disposition=request.headers.get(
'Content-Disposition: attachment;'),
content_encoding=request.headers.get('Content-Encoding'),
content_language=request.headers.get('Content-Language'),
content_type=request.headers.get('Content-Type'))
url = file_service.make_file_url(share_name,
directory_name,
file_name,
protocol='https',
sas_token=sas_token)
return Response(response=url + "",
status=200,
content_type='text/plain')
except Exception as e:
logger.exception(e)
return abort(500, e)
def saveModel(customer, modelName, model, storage_account_name,
storage_account_key):
fileService = FileService(account_name=storage_account_name,
account_key=storage_account_key)
if not fileService.exists('trainedmodels', customer):
fileService.create_share('trainedmodels')
fileService.create_directory('trainedmodels', customer)
if not fileService.exists('trainedmodels', customer + '/' + modelName):
fileService.create_directory('trainedmodels',
customer + '/' + modelName)
modelPickle = pickle.dumps(model)
timestr = time.strftime('%Y%m%d-%H%M%S')
fileName = modelName + '_' + timestr + '.pkl'
fileService.create_file_from_bytes('trainedmodels',
customer + '/' + modelName, fileName,
modelPickle)
print(fileName + ' saved.')
RECEIPTS_DIRECTORY = 'delivery-receipts'
RECEIPTS_URL = ('http://www.cmegroup.com/delivery_reports/'
'deliverable-commodities-under-registration.xls')
RECEIPTS_FILENAME_SUFFIX = '-deliverable-commodities-under-registration.xls'
STOCKS_DIRECTORY = 'stocks-of-grain'
STOCKS_URL = 'http://www.cmegroup.com/delivery_reports/stocks-of-grain-updated-tuesday.xls'
STOCKS_FILENAME_SUFFIX = '-stocks-of-grain-updated-tuesday.xls'
def filename(suffix):
return datetime.now().strftime('%Y%m%d') + suffix
def get_bytes(url):
return requests.get(url).content
f = FileService(account_name=ACCOUNT_NAME, account_key=KEY)
f.create_share(SHARE_NAME)
f.create_directory(SHARE_NAME, RECEIPTS_DIRECTORY)
f.create_file_from_bytes(SHARE_NAME, RECEIPTS_DIRECTORY,
filename(RECEIPTS_FILENAME_SUFFIX),
get_bytes(RECEIPTS_URL))
f.create_directory(SHARE_NAME, STOCKS_DIRECTORY)
f.create_file_from_bytes(SHARE_NAME, STOCKS_DIRECTORY,
filename(STOCKS_FILENAME_SUFFIX),
get_bytes(STOCKS_URL))
class AzureHandler(Handler):
class Settings(BaseSettings):
client_id: UUID
client_secret: SecretStr
keyvault_url: HttpUrl
storage_blob_url: HttpUrl
tenant_id: UUID
storage_name: str
storage_share_name: str
storage_sas: SecretStr
def __init__(self, env_file) -> None:
self.settings = AzureHandler.Settings(_env_file=env_file)
self.credential = ClientSecretCredential(
str(self.settings.tenant_id),
str(self.settings.client_id),
self.settings.client_secret.get_secret_value(),
)
# Create File and KeyVault Secret Clients
self.secret_index = {}
self.secretsclient = SecretClient(self.settings.keyvault_url,
self.credential)
self.fileclient = FileService(
self.settings.storage_name,
sas_token=self.settings.storage_sas.get_secret_value(),
)
try:
self.fileclient.create_share("lootmarshal")
self.fileclient.create_directory("lootmarshal", "binary_dumps")
asyncio.create_task(self.build_secret_index())
except Exception as e:
raise e
async def validate(self) -> bool:
try:
if not len(self.fileclient.list_shares(num_results=1)):
raise Exception(
"Azure FileService Client could not list shares! Verify authentication."
)
[p async for p in self.secretsclient.list_properties_of_secrets()]
return True
except Exception as e:
raise e
async def build_secret_index(self):
while True:
try:
self.secret_index = [{
"name": p.name,
"content_type": p.content_type,
"tags": p.tags
} async for p in self.secretsclient.list_properties_of_secrets(
)]
except Exception as e:
raise e
await asyncio.sleep(10)
def write_file(self, directory: str, name: str, file: bytes):
name = f"{name}_{time.strftime('%Y%m%d-%H%M%S')}"
logging.info(
f"Writing {name} ({len(file)} bytes) to {directory}/{name}.")
self.fileclient.create_file_from_bytes(
self.settings.storage_share_name, directory, name, file)
async def read_secret(self, name) -> dict:
try:
secret = await self.secretsclient.get_secret(name)
formatted = {
"name": secret.name,
"value": secret.value,
"content_type": secret.properties.content_type,
"tags": secret.properties.tags,
}
return formatted
except Exception as e:
raise HTTPException(status_code=404, detail=str(e))
async def write_secret(self, name, value, content_type, tags):
try:
return await self.secretsclient.set_secret(
name, value, content_type=content_type, tags=tags)
except Exception as e:
raise HTTPException(status_code=404, detail=str(e))
async def list_secrets(self):
try:
secret_list = []
async for props in self.secretsclient.list_properties_of_secrets():
secret = await self.secretsclient.get_secret(props.name)
formatted = {
"name": secret.name,
"value": secret.value,
"content_type": secret.properties.content_type,
"tags": secret.properties.tags,
}
secret_list.append(formatted)
return json.dumps(secret_list, indent=4)
except Exception as e:
raise HTTPException(status_code=404, detail=str(e))
