def test_using_directory_sas_to_read(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return client = self._create_directory_and_get_directory_client() directory_name = client.path_name # generate a token with directory level read permission token = generate_directory_sas( self.dsc.account_name, self.file_system_name, directory_name, account_key=self.dsc.credential.account_key, permission=DirectorySasPermissions(read=True), expiry=datetime.utcnow() + timedelta(hours=1), ) directory_client = DataLakeDirectoryClient(self.dsc.url, self.file_system_name, directory_name, credential=token) access_control = directory_client.get_access_control() self.assertIsNotNone(access_control)
def __upload_file(directory_client: DataLakeDirectoryClient, filename: str, file_data: bytes): # NOTE: Using get_file_client as a context manager will close the parent DirectoryClient on __exit__ file_client = directory_client.get_file_client(filename) try: if len(file_data) <= 0: message = f'ERROR: Unexpected error - File size 0 (or negative) - NOT POSSIBLE ({len(file_data)})' raise HTTPException(status_code=HTTPStatus.NOT_ACCEPTABLE, detail=message) file_client.upload_data(file_data, overwrite=True) # Check if file is written as expected: due to the 0-byte file error verification_client = directory_client.get_file_client(filename) file_properties = verification_client.get_file_properties() if file_properties.size != len(file_data): message = f'ERROR: Azure write of ({file_client.path_name}) upload: {len(file_data)}' + \ f' stored: {file_properties.size} last_modified: {file_properties.last_modified}' raise HTTPException(status_code=HTTPStatus.NOT_ACCEPTABLE, detail=message) # End check except ResourceModifiedError as error: message = f'({type(error).__name__}) Upload failed (blob modified while writing): {error}' raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail=message) from error except HttpResponseError as error: message = f'({type(error).__name__}) An error occurred while uploading file: {error}' raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail=message) from error
def test_using_oauth_token_credential_to_create_directory(self): # generate a token with directory level create permission directory_name = self._get_directory_reference() token_credential = self.generate_oauth_token() directory_client = DataLakeDirectoryClient(self.dsc.url, self.file_system_name, directory_name, credential=token_credential) response = directory_client.create_directory() self.assertIsNotNone(response)
def test_file_sas_only_applies_to_file_level(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # SAS URL is calculated from storage key, so this test runs live only file_name = self._get_file_reference() directory_name = self._get_directory_reference() self._create_file_and_return_client(directory=directory_name, file=file_name) # generate a token with file level read and write permissions token = generate_file_sas( self.dsc.account_name, self.file_system_name, directory_name, file_name, self.dsc.credential.account_key, permission=FileSasPermissions(read=True, write=True), expiry=datetime.utcnow() + timedelta(hours=1), ) # read the created file which is under root directory file_client = DataLakeFileClient(self.dsc.url, self.file_system_name, directory_name + '/' + file_name, credential=token) properties = file_client.get_file_properties() # make sure we can read the file properties self.assertIsNotNone(properties) # try to write to the created file with the token response = file_client.append_data(b"abcd", 0, 4, validate_content=True) self.assertIsNotNone(response) # the token is for file level, so users are not supposed to have access to file system level operations file_system_client = FileSystemClient(self.dsc.url, self.file_system_name, credential=token) with self.assertRaises(ClientAuthenticationError): file_system_client.get_file_system_properties() # the token is for file level, so users are not supposed to have access to directory level operations directory_client = DataLakeDirectoryClient(self.dsc.url, self.file_system_name, directory_name, credential=token) with self.assertRaises(ClientAuthenticationError): directory_client.get_directory_properties()
def __check_directory_exist(directory_client: DataLakeDirectoryClient): try: directory_client.get_directory_properties() except ResourceNotFoundError as error: message = f'({type(error).__name__}) The given dataset doesnt exist: {error}' raise HTTPException(status_code=error.status_code, detail=message) from error except ClientAuthenticationError as error: message = f'({type(error).__name__}) You do not have permission to access the dataset: {error}' raise HTTPException(status_code=error.status_code, detail=message) from error except HttpResponseError as error: message = f'({type(error).__name__}) An error occurred while checking if the dataset exist: {error}' raise HTTPException(status_code=error.status_code, detail=message) from error
def __get_destination_directory_client_based_on_event_time(directory_client: DataLakeDirectoryClient, event_time: str) -> DataLakeDirectoryClient: event_time_obj, time_resolution = parse_date_str(event_time) path = get_file_path_with_respect_to_time_resolution(event_time_obj, time_resolution, '') return directory_client.get_sub_directory_client(path)
def __download_file(filename: str, directory_client: DataLakeDirectoryClient) -> StorageStreamDownloader: file_client = directory_client.get_file_client(filename) try: downloaded_file = file_client.download_file() return downloaded_file except ResourceNotFoundError as error: message = f'({type(error).__name__}) The given dataset doesnt exist: {error}' raise HTTPException(status_code=error.status_code, detail=message) from error except HttpResponseError as error: message = f'({type(error).__name__}) File could not be downloaded: {error}' raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail=message) from error
def test_using_directory_sas_to_create(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # generate a token with directory level create permission directory_name = self._get_directory_reference() token = generate_directory_sas( self.dsc.account_name, self.file_system_name, directory_name, account_key=self.dsc.credential.account_key, permission=DirectorySasPermissions(create=True), expiry=datetime.utcnow() + timedelta(hours=1), ) directory_client = DataLakeDirectoryClient(self.dsc.url, self.file_system_name, directory_name, credential=token) response = directory_client.create_directory() self.assertIsNotNone(response)
def test_rename_dir_with_file_system_sas(self): if TestMode.need_recording_file(self.test_mode): return token = generate_file_system_sas( self.dsc.account_name, self.file_system_name, self.dsc.credential.account_key, FileSystemSasPermissions(write=True, read=True, delete=True), datetime.utcnow() + timedelta(hours=1), ) # read the created file which is under root directory dir_client = DataLakeDirectoryClient(self.dsc.url, self.file_system_name, "olddirectory", credential=token) dir_client.create_directory() new_client = dir_client.rename_directory(dir_client.file_system_name + '/' + 'newdirectory') new_client.get_directory_properties() self.assertEqual(new_client.path_name, "newdirectory")
def __validate_json_with_schema(directory_client: DataLakeDirectoryClient, json_schema_file_path: str, data_dict: Dict): file_client = directory_client.get_file_client(json_schema_file_path) schema = __get_validation_schema(file_client) try: fastjsonschema.validate(schema, data_dict) except (TypeError, fastjsonschema.JsonSchemaDefinitionException, AttributeError) as error: raise HTTPException( status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail=f'Invalid schema definition: {getattr(error, "message", error)}') from error except fastjsonschema.JsonSchemaValueException as error: raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail={ 'message': f'JSON Schema validation error: {error.message}', 'name': f'{error.name}', 'rule': f'{error.rule}', 'rule_definition': f'{error.rule_definition}' }) from error
def __get_directory_client(token: str, guid: str) -> DataLakeDirectoryClient: account_url = config['Azure Storage']['account_url'] filesystem_name = config['Azure Storage']['filesystem_name'] credential = AzureCredential(token) return DataLakeDirectoryClient(account_url, filesystem_name, guid, credential=credential)
def __get_destination_directory_client_based_on_ingress_time(directory_client: DataLakeDirectoryClient) \ -> DataLakeDirectoryClient: now = datetime.utcnow() path = f'year={now.year:02d}/month={now.month:02d}/day={now.day:02d}/hour={now.hour:02d}' return directory_client.get_sub_directory_client(path)
def instantiate_directory_client_from_conn_str(): # [START instantiate_directory_client_from_conn_str] from azure.storage.filedatalake import DataLakeDirectoryClient DataLakeDirectoryClient.from_connection_string(connection_string, "myfilesystem", "mydirectory")