async def test_list_paths_using_file_sys_delegation_sas_async(
self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
url = self._get_account_url(datalake_storage_account_name)
token_credential = self.generate_oauth_token()
dsc = DataLakeServiceClient(url, token_credential)
file_system_name = self._get_file_system_reference()
directory_client_name = '/'
directory_client = (await dsc.create_file_system(file_system_name)).get_directory_client(directory_client_name)
random_guid = uuid.uuid4()
await directory_client.set_access_control(owner=random_guid, permissions='0777')
delegation_key = await dsc.get_user_delegation_key(datetime.utcnow(),
datetime.utcnow() + timedelta(hours=1))
token = generate_file_system_sas(
dsc.account_name,
file_system_name,
delegation_key,
permission=DirectorySasPermissions(list=True),
expiry=datetime.utcnow() + timedelta(hours=1),
agent_object_id=random_guid
)
sas_directory_client = FileSystemClient(self.dsc.url, file_system_name,
credential=token)
paths = list()
async for path in sas_directory_client.get_paths():
paths.append(path)
self.assertEqual(0, 0)
async def look_up_parquet_files(egress_folder_path: str,
filesystem_client: FileSystemClient,
directory_client: DataLakeDirectoryClient):
"""
Args:
egress_folder_path: Path (without guid prefix) to folder in which parquet files are placed.
Example "year=2021/month=01"
Returns:
List with paths to parquet files in egress_folder_path (without guid prefix).
"""
guid = directory_client.path_name
path_with_guid = f'{guid}/{egress_folder_path}'
paths_in_directory = filesystem_client.get_paths(path=path_with_guid)
try:
parquet_paths_with_guid = [
path.name async for path in paths_in_directory
if path.name.endswith('.parquet')
]
except ResourceNotFoundError:
return []
# Remove guid from paths
parquet_paths = [p.split('/', 1)[1] for p in parquet_paths_with_guid]
return parquet_paths
async def create_file_from_file_system(self):
# [START create_file_system_client_from_connection_string]
from azure.storage.filedatalake.aio import FileSystemClient
file_system_client = FileSystemClient.from_connection_string(
self.connection_string, "filesystemforcreate")
# [END create_file_system_client_from_connection_string]
async with file_system_client:
await file_system_client.create_file_system()
# [START create_directory_from_file_system]
directory_client = await file_system_client.create_directory(
"mydirectory")
# [END create_directory_from_file_system]
# [START create_file_from_file_system]
file_client = await file_system_client.create_file("myfile")
# [END create_file_from_file_system]
# [START delete_file_from_file_system]
await file_system_client.delete_file("myfile")
# [END delete_file_from_file_system]
# [START delete_directory_from_file_system]
await file_system_client.delete_directory("mydirectory")
# [END delete_directory_from_file_system]
await file_system_client.delete_file_system()
async def batch_delete_files_or_empty_directories(self):
from azure.storage.filedatalake.aio import FileSystemClient
file_system_client = FileSystemClient.from_connection_string(
self.connection_string, "filesystemforcreate")
async with file_system_client:
await file_system_client.create_file_system()
data = b'hello world'
try:
# create file1
await file_system_client.get_file_client('file1').upload_data(
data, overwrite=True)
# create file2, then pass file properties in batch delete later
file2 = file_system_client.get_file_client('file2')
await file2.upload_data(data, overwrite=True)
file2_properties = await file2.get_file_properties()
# create file3 and batch delete it later only etag matches this file3 etag
file3 = file_system_client.get_file_client('file3')
await file3.upload_data(data, overwrite=True)
file3_props = await file3.get_file_properties()
file3_etag = file3_props.etag
# create dir1
# empty directory can be deleted using delete_files
await file_system_client.get_directory_client(
'dir1').create_directory(),
# create dir2, then pass directory properties in batch delete later
dir2 = file_system_client.get_directory_client('dir2')
await dir2.create_directory()
dir2_properties = await dir2.get_directory_properties()
except:
pass
# Act
response = await self._to_list(await
file_system_client.delete_files(
'file1',
file2_properties, {
'name': 'file3',
'etag': file3_etag
},
'dir1',
dir2_properties,
raise_on_any_failure=False))
print("total number of sub-responses:" + len(response))
print(response[0].status_code)
print(response[2].status_code)
print(response[3].status_code)
async def __get_file_stream_for_ikontrol_file(
file_path: str, filesystem_client: FileSystemClient) -> BytesIO:
guid = config['iKontrol']['guid']
directory_client = filesystem_client.get_directory_client(guid)
await check_directory_exist(directory_client)
file_download = await download_file(file_path, directory_client)
file_content = await file_download.readall()
stream = BytesIO(file_content)
return stream
async def _test_file_sas_only_applies_to_file_level(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
file_name = self._get_file_reference()
directory_name = self._get_directory_reference()
await self._create_file_and_return_client(directory=directory_name,
file=file_name)
# generate a token with file level read and write permissions
token = generate_file_sas(
self.dsc.account_name,
self.file_system_name,
directory_name,
file_name,
self.dsc.credential.account_key,
permission=FileSasPermissions(read=True, write=True),
expiry=datetime.utcnow() + timedelta(hours=1),
)
# read the created file which is under root directory
file_client = DataLakeFileClient(self.dsc.url,
self.file_system_name,
directory_name + '/' + file_name,
credential=token)
properties = await file_client.get_file_properties()
# make sure we can read the file properties
self.assertIsNotNone(properties)
# try to write to the created file with the token
response = await file_client.append_data(b"abcd",
0,
4,
validate_content=True)
self.assertIsNotNone(response)
# the token is for file level, so users are not supposed to have access to file system level operations
file_system_client = FileSystemClient(self.dsc.url,
self.file_system_name,
credential=token)
with self.assertRaises(ClientAuthenticationError):
await file_system_client.get_file_system_properties()
# the token is for file level, so users are not supposed to have access to directory level operations
directory_client = DataLakeDirectoryClient(self.dsc.url,
self.file_system_name,
directory_name,
credential=token)
with self.assertRaises(ClientAuthenticationError):
await directory_client.get_directory_properties()
async def get_filesystem_client(token: str) -> FileSystemClient:
"""
Returns a FileSystemClient initialized from config and security access token.
Args:
token (str): Security Access Token
Returns:
azure.storage.filedatalake.aio.FileSystemClient initialized from config and token.
"""
account_url = config['Azure Storage']['account_url']
filesystem_name = config['Azure Storage']['filesystem_name']
credential = AzureCredentialAIO(token)
return FileSystemClient(account_url,
filesystem_name,
credential=credential)