def post_file(share_name, path_to_file): try: account_name, account_key = get_auth(request.authorization) file_service = FileService(account_name=account_name, account_key=account_key) start_timedelta, expiry_timedelta = get_sas_params(request.args) directory_name, file_name = get_location(path_to_file) if request.headers.get('Transfer-Encoding') == 'chunked': file_service.create_file_from_stream(share_name, directory_name, file_name, request.stream, count=4096) else: file_service.create_file_from_bytes(share_name, directory_name, file_name, request.get_data()) sas_token = file_service.generate_file_shared_access_signature( share_name, directory_name=directory_name, file_name=file_name, permission=SharePermissions(read=True), expiry=datetime.now() + expiry_timedelta, start=start_timedelta, id=None, ip=None, protocol='https', cache_control=request.headers.get('Cache-Control'), content_disposition=request.headers.get( 'Content-Disposition: attachment;'), content_encoding=request.headers.get('Content-Encoding'), content_language=request.headers.get('Content-Language'), content_type=request.headers.get('Content-Type')) url = file_service.make_file_url(share_name, directory_name, file_name, protocol='https', sas_token=sas_token) return Response(response=url + "", status=200, content_type='text/plain') except Exception as e: logger.exception(e) return abort(500, e)
def saveModel(customer, modelName, model, storage_account_name, storage_account_key): fileService = FileService(account_name=storage_account_name, account_key=storage_account_key) if not fileService.exists('trainedmodels', customer): fileService.create_share('trainedmodels') fileService.create_directory('trainedmodels', customer) if not fileService.exists('trainedmodels', customer + '/' + modelName): fileService.create_directory('trainedmodels', customer + '/' + modelName) modelPickle = pickle.dumps(model) timestr = time.strftime('%Y%m%d-%H%M%S') fileName = modelName + '_' + timestr + '.pkl' fileService.create_file_from_bytes('trainedmodels', customer + '/' + modelName, fileName, modelPickle) print(fileName + ' saved.')
RECEIPTS_DIRECTORY = 'delivery-receipts' RECEIPTS_URL = ('http://www.cmegroup.com/delivery_reports/' 'deliverable-commodities-under-registration.xls') RECEIPTS_FILENAME_SUFFIX = '-deliverable-commodities-under-registration.xls' STOCKS_DIRECTORY = 'stocks-of-grain' STOCKS_URL = 'http://www.cmegroup.com/delivery_reports/stocks-of-grain-updated-tuesday.xls' STOCKS_FILENAME_SUFFIX = '-stocks-of-grain-updated-tuesday.xls' def filename(suffix): return datetime.now().strftime('%Y%m%d') + suffix def get_bytes(url): return requests.get(url).content f = FileService(account_name=ACCOUNT_NAME, account_key=KEY) f.create_share(SHARE_NAME) f.create_directory(SHARE_NAME, RECEIPTS_DIRECTORY) f.create_file_from_bytes(SHARE_NAME, RECEIPTS_DIRECTORY, filename(RECEIPTS_FILENAME_SUFFIX), get_bytes(RECEIPTS_URL)) f.create_directory(SHARE_NAME, STOCKS_DIRECTORY) f.create_file_from_bytes(SHARE_NAME, STOCKS_DIRECTORY, filename(STOCKS_FILENAME_SUFFIX), get_bytes(STOCKS_URL))
class AzureHandler(Handler): class Settings(BaseSettings): client_id: UUID client_secret: SecretStr keyvault_url: HttpUrl storage_blob_url: HttpUrl tenant_id: UUID storage_name: str storage_share_name: str storage_sas: SecretStr def __init__(self, env_file) -> None: self.settings = AzureHandler.Settings(_env_file=env_file) self.credential = ClientSecretCredential( str(self.settings.tenant_id), str(self.settings.client_id), self.settings.client_secret.get_secret_value(), ) # Create File and KeyVault Secret Clients self.secret_index = {} self.secretsclient = SecretClient(self.settings.keyvault_url, self.credential) self.fileclient = FileService( self.settings.storage_name, sas_token=self.settings.storage_sas.get_secret_value(), ) try: self.fileclient.create_share("lootmarshal") self.fileclient.create_directory("lootmarshal", "binary_dumps") asyncio.create_task(self.build_secret_index()) except Exception as e: raise e async def validate(self) -> bool: try: if not len(self.fileclient.list_shares(num_results=1)): raise Exception( "Azure FileService Client could not list shares! Verify authentication." ) [p async for p in self.secretsclient.list_properties_of_secrets()] return True except Exception as e: raise e async def build_secret_index(self): while True: try: self.secret_index = [{ "name": p.name, "content_type": p.content_type, "tags": p.tags } async for p in self.secretsclient.list_properties_of_secrets( )] except Exception as e: raise e await asyncio.sleep(10) def write_file(self, directory: str, name: str, file: bytes): name = f"{name}_{time.strftime('%Y%m%d-%H%M%S')}" logging.info( f"Writing {name} ({len(file)} bytes) to {directory}/{name}.") self.fileclient.create_file_from_bytes( self.settings.storage_share_name, directory, name, file) async def read_secret(self, name) -> dict: try: secret = await self.secretsclient.get_secret(name) formatted = { "name": secret.name, "value": secret.value, "content_type": secret.properties.content_type, "tags": secret.properties.tags, } return formatted except Exception as e: raise HTTPException(status_code=404, detail=str(e)) async def write_secret(self, name, value, content_type, tags): try: return await self.secretsclient.set_secret( name, value, content_type=content_type, tags=tags) except Exception as e: raise HTTPException(status_code=404, detail=str(e)) async def list_secrets(self): try: secret_list = [] async for props in self.secretsclient.list_properties_of_secrets(): secret = await self.secretsclient.get_secret(props.name) formatted = { "name": secret.name, "value": secret.value, "content_type": secret.properties.content_type, "tags": secret.properties.tags, } secret_list.append(formatted) return json.dumps(secret_list, indent=4) except Exception as e: raise HTTPException(status_code=404, detail=str(e))