async def test_service_client_session_closes_after_filesystem_creation(
self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# Arrange
dsc2 = DataLakeServiceClient(self.dsc.url, credential=datalake_storage_account_key)
async with DataLakeServiceClient(
self.dsc.url, credential=datalake_storage_account_key) as ds_client:
fs1 = await ds_client.create_file_system(self._get_file_system_reference(prefix="fs1"))
await fs1.delete_file_system()
await dsc2.create_file_system(self._get_file_system_reference(prefix="fs2"))
await dsc2.close()
async def test_list_paths_using_file_sys_delegation_sas_async(
self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
url = self._get_account_url(datalake_storage_account_name)
token_credential = self.generate_oauth_token()
dsc = DataLakeServiceClient(url, token_credential)
file_system_name = self._get_file_system_reference()
directory_client_name = '/'
directory_client = (await dsc.create_file_system(file_system_name)).get_directory_client(directory_client_name)
random_guid = uuid.uuid4()
await directory_client.set_access_control(owner=random_guid, permissions='0777')
delegation_key = await dsc.get_user_delegation_key(datetime.utcnow(),
datetime.utcnow() + timedelta(hours=1))
token = generate_file_system_sas(
dsc.account_name,
file_system_name,
delegation_key,
permission=DirectorySasPermissions(list=True),
expiry=datetime.utcnow() + timedelta(hours=1),
agent_object_id=random_guid
)
sas_directory_client = FileSystemClient(self.dsc.url, file_system_name,
credential=token)
paths = list()
async for path in sas_directory_client.get_paths():
paths.append(path)
self.assertEqual(0, 0)
async def test_get_access_control_using_delegation_sas_async(
self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
url = self._get_account_url(datalake_storage_account_name)
token_credential = self.generate_oauth_token()
dsc = DataLakeServiceClient(url, token_credential, logging_enable=True)
file_system_name = self._get_file_system_reference()
directory_client_name = '/'
(await dsc.create_file_system(file_system_name)).get_directory_client(directory_client_name)
directory_client = self.dsc.get_directory_client(file_system_name, directory_client_name)
random_guid = uuid.uuid4()
await directory_client.set_access_control(owner=random_guid,
permissions='0777')
acl = await directory_client.get_access_control()
delegation_key = await dsc.get_user_delegation_key(datetime.utcnow(),
datetime.utcnow() + timedelta(hours=1))
token = generate_file_system_sas(
dsc.account_name,
file_system_name,
delegation_key,
permission=FileSystemSasPermissions(
read=True, execute=True, manage_access_control=True, manage_ownership=True),
expiry=datetime.utcnow() + timedelta(hours=1),
agent_object_id=random_guid
)
sas_directory_client = DataLakeDirectoryClient(self.dsc.url, file_system_name, directory_client_name,
credential=token, logging_enable=True)
access_control = await sas_directory_client.get_access_control()
self.assertIsNotNone(access_control)
def setUp(self):
super(LargeFileTest, self).setUp()
url = self._get_account_url()
self.payload_dropping_policy = PayloadDroppingPolicy()
credential_policy = _format_shared_key_credential(
self.settings.STORAGE_DATA_LAKE_ACCOUNT_NAME,
self.settings.STORAGE_DATA_LAKE_ACCOUNT_KEY)
self.dsc = DataLakeServiceClient(
url,
credential=self.settings.STORAGE_DATA_LAKE_ACCOUNT_KEY,
_additional_pipeline_policies=[
self.payload_dropping_policy, credential_policy
])
loop = asyncio.get_event_loop()
loop.run_until_complete(self.dsc.__aenter__())
self.config = self.dsc._config
self.file_system_name = self.get_resource_name('filesystem')
if not self.is_playback():
file_system = self.dsc.get_file_system_client(
self.file_system_name)
try:
loop = asyncio.get_event_loop()
loop.run_until_complete(
file_system.create_file_system(timeout=5))
except ResourceExistsError:
pass
async def test_restore_file_system_with_sas(self, datalake_storage_account_name, datalake_storage_account_key):
pytest.skip(
"We are generating a SAS token therefore play only live but we also need a soft delete enabled account.")
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
token = generate_account_sas(
self.dsc.account_name,
self.dsc.credential.account_key,
ResourceTypes(service=True, file_system=True),
AccountSasPermissions(read=True, write=True, list=True, delete=True),
datetime.utcnow() + timedelta(hours=1),
)
dsc = DataLakeServiceClient(self.dsc.url, token)
name = self._get_file_system_reference(prefix="filesystem")
filesystem_client = await dsc.create_file_system(name)
await filesystem_client.delete_file_system()
# to make sure the filesystem is deleted
with self.assertRaises(ResourceNotFoundError):
await filesystem_client.get_file_system_properties()
filesystem_list = []
async for fs in self.dsc.list_file_systems(include_deleted=True):
filesystem_list.append(fs)
self.assertTrue(len(filesystem_list) >= 1)
restored_version = 0
for filesystem in filesystem_list:
# find the deleted filesystem and restore it
if filesystem.deleted and filesystem.name == filesystem_client.file_system_name:
restored_fs_client = await dsc.undelete_file_system(filesystem.name, filesystem.deleted_version,
new_name="restored" + name + str(restored_version))
restored_version += 1
# to make sure the deleted filesystem is restored
props = await restored_fs_client.get_file_system_properties()
self.assertIsNotNone(props)
async def run():
account_name = os.getenv('STORAGE_ACCOUNT_NAME', "")
account_key = os.getenv('STORAGE_ACCOUNT_KEY', "")
# set up the service client with the credentials from the environment variables
service_client = DataLakeServiceClient(
account_url="{}://{}.dfs.core.windows.net".format(
"https", account_name),
credential=account_key)
async with service_client:
# generate a random name for testing purpose
fs_name = "testfs{}".format(random.randint(1, 1000))
print("Generating a test filesystem named '{}'.".format(fs_name))
# create the filesystem
filesystem_client = await service_client.create_file_system(
file_system=fs_name)
# invoke the sample code
try:
await directory_sample(filesystem_client)
finally:
# clean up the demo filesystem
await filesystem_client.delete_file_system()
async def _test_read_file_with_user_delegation_key(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Create file
file_client = await self._create_file_and_return_client()
data = self.get_random_bytes(1024)
# Upload data to file
await file_client.append_data(data, 0, len(data))
await file_client.flush_data(len(data))
# Get user delegation key
token_credential = self.generate_async_oauth_token()
service_client = DataLakeServiceClient(self._get_oauth_account_url(), credential=token_credential)
user_delegation_key = await service_client.get_user_delegation_key(datetime.utcnow(),
datetime.utcnow() + timedelta(hours=1))
sas_token = generate_file_sas(file_client.account_name,
file_client.file_system_name,
None,
file_client.path_name,
user_delegation_key=user_delegation_key,
permission=FileSasPermissions(read=True, create=True, write=True, delete=True),
expiry=datetime.utcnow() + timedelta(hours=1),
)
# doanload the data and make sure it is the same as uploaded data
new_file_client = DataLakeFileClient(self._get_account_url(),
file_client.file_system_name,
file_client.path_name,
credential=sas_token)
downloaded_data = await new_file_client.read_file()
self.assertEqual(data, downloaded_data)
async def _test_service_client_session_closes_after_filesystem_creation(
self):
# Arrange
dsc2 = DataLakeServiceClient(
self.dsc.url,
credential=self.settings.STORAGE_DATA_LAKE_ACCOUNT_KEY)
async with DataLakeServiceClient(
self.dsc.url,
credential=self.settings.STORAGE_DATA_LAKE_ACCOUNT_KEY
) as ds_client:
fs1 = await ds_client.create_file_system(
self._get_file_system_reference(prefix="fs1"))
await fs1.delete_file_system()
await dsc2.create_file_system(
self._get_file_system_reference(prefix="fs2"))
await dsc2.close()
def _setUp(self, account_name, account_key):
url = self.account_url(account_name, 'dfs')
self.dsc = DataLakeServiceClient(url,
credential=account_key,
transport=AiohttpTestTransport(),
logging_enable=True)
self.config = self.dsc._config
self.test_file_systems = []
def setUp(self):
super(FileSystemTest, self).setUp()
url = self._get_account_url()
self.dsc = DataLakeServiceClient(url, credential=self.settings.STORAGE_DATA_LAKE_ACCOUNT_KEY,
transport=AiohttpTestTransport())
self.config = self.dsc._config
loop = asyncio.get_event_loop()
loop.run_until_complete(self.dsc.__aenter__())
self.test_file_systems = []
async def _setUp(self, account_name, account_key):
url = self.account_url(account_name, 'dfs')
self.dsc = DataLakeServiceClient(url, credential=account_key)
self.config = self.dsc._config
self.file_system_name = self.get_resource_name('filesystem')
if not self.is_playback():
file_system = self.dsc.get_file_system_client(self.file_system_name)
try:
await file_system.create_file_system(timeout=5)
except ResourceExistsError:
pass
async def test_list_system_filesystems_async(self,
datalake_storage_account_name,
datalake_storage_account_key):
self._setUp(datalake_storage_account_name,
datalake_storage_account_key)
# Arrange
dsc = DataLakeServiceClient(self.dsc.url,
credential=datalake_storage_account_key)
# Act
filesystems = []
async for fs in dsc.list_file_systems(include_system=True):
filesystems.append(fs)
# Assert
found = False
for fs in filesystems:
if fs.name == "$logs":
found = True
self.assertEqual(found, True)
def setUp(self):
super(DirectoryTest, self).setUp()
url = self._get_account_url()
self.dsc = DataLakeServiceClient(url, credential=self.settings.STORAGE_DATA_LAKE_ACCOUNT_KEY,
transport=AiohttpTestTransport())
loop = asyncio.get_event_loop()
loop.run_until_complete(self.dsc.__aenter__())
self.config = self.dsc._config
self.file_system_name = self.get_resource_name('filesystem')
if not self.is_playback():
file_system = self.dsc.get_file_system_client(self.file_system_name)
try:
loop.run_until_complete(file_system.create_file_system(timeout=5))
except ResourceExistsError:
pass
async def _setUp(self, account_name, account_key):
url = self._get_account_url(account_name)
self.payload_dropping_policy = PayloadDroppingPolicy()
credential_policy = _format_shared_key_credential(account_name, account_key)
self.dsc = DataLakeServiceClient(url,
credential=account_key,
_additional_pipeline_policies=[self.payload_dropping_policy, credential_policy])
self.config = self.dsc._config
self.file_system_name = self.get_resource_name('filesystem')
if not self.is_playback():
file_system = self.dsc.get_file_system_client(self.file_system_name)
try:
await file_system.create_file_system(timeout=5)
except ResourceExistsError:
pass
def _setUp(self, account_name, account_key):
url = self._get_account_url(account_name)
self.dsc = DataLakeServiceClient(url, credential=account_key, logging_enable=True)
self.config = self.dsc._config
async def data_lake_service_sample():
# Instantiate a DataLakeServiceClient using a connection string
# [START create_datalake_service_client]
from azure.storage.filedatalake.aio import DataLakeServiceClient
datalake_service_client = DataLakeServiceClient.from_connection_string(
connection_string)
# [END create_datalake_service_client]
# Instantiate a DataLakeServiceClient Azure Identity credentials.
# [START create_datalake_service_client_oauth]
from azure.identity.aio import ClientSecretCredential
token_credential = ClientSecretCredential(
active_directory_tenant_id,
active_directory_application_id,
active_directory_application_secret,
)
datalake_service_client = DataLakeServiceClient(
"https://{}.dfs.core.windows.net".format(account_name),
credential=token_credential)
# [END create_datalake_service_client_oauth]
async with datalake_service_client:
# get user delegation key
# [START get_user_delegation_key]
from datetime import datetime, timedelta
user_delegation_key = await datalake_service_client.get_user_delegation_key(
datetime.utcnow(),
datetime.utcnow() + timedelta(hours=1))
# [END get_user_delegation_key]
# Create file systems
# [START create_file_system_from_service_client]
await datalake_service_client.create_file_system("filesystem")
# [END create_file_system_from_service_client]
file_system_client = await datalake_service_client.create_file_system(
"anotherfilesystem")
# List file systems
# [START list_file_systems]
file_systems = datalake_service_client.list_file_systems()
async for file_system in file_systems:
print(file_system.name)
# [END list_file_systems]
# Get Clients from DataLakeServiceClient
file_system_client = datalake_service_client.get_file_system_client(
file_system_client.file_system_name)
# [START get_directory_client_from_service_client]
directory_client = datalake_service_client.get_directory_client(
file_system_client.file_system_name, "mydirectory")
# [END get_directory_client_from_service_client]
# [START get_file_client_from_service_client]
file_client = datalake_service_client.get_file_client(
file_system_client.file_system_name, "myfile")
# [END get_file_client_from_service_client]
# Create file and set properties
metadata = {'hello': 'world', 'number': '42'}
from azure.storage.filedatalake import ContentSettings
content_settings = ContentSettings(content_language='spanish',
content_disposition='inline')
await file_client.create_file(content_settings=content_settings)
await file_client.set_metadata(metadata=metadata)
file_props = await file_client.get_file_properties()
print(file_props.metadata)
# Create file/directory and set properties
await directory_client.create_directory(
content_settings=content_settings, metadata=metadata)
dir_props = await directory_client.get_directory_properties()
print(dir_props.metadata)
# Delete File Systems
# [START delete_file_system_from_service_client]
await datalake_service_client.delete_file_system("filesystem")
# [END delete_file_system_from_service_client]
await file_system_client.delete_file_system()
await token_credential.close()
