def test_restore_file_system_with_sas(self, datalake_storage_account_name, datalake_storage_account_key):
pytest.skip(
"We are generating a SAS token therefore play only live but we also need a soft delete enabled account.")
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
token = generate_account_sas(
self.dsc.account_name,
self.dsc.credential.account_key,
ResourceTypes(service=True, file_system=True),
AccountSasPermissions(read=True, write=True, list=True, delete=True),
datetime.utcnow() + timedelta(hours=1),
)
dsc = DataLakeServiceClient(self.dsc.url, token)
name = self._get_file_system_reference(prefix="filesystem")
filesystem_client = dsc.create_file_system(name)
filesystem_client.delete_file_system()
# to make sure the filesystem is deleted
with self.assertRaises(ResourceNotFoundError):
filesystem_client.get_file_system_properties()
filesystem_list = list(dsc.list_file_systems(include_deleted=True))
self.assertTrue(len(filesystem_list) >= 1)
restored_version = 0
for filesystem in filesystem_list:
# find the deleted filesystem and restore it
if filesystem.deleted and filesystem.name == filesystem_client.file_system_name:
restored_fs_client = dsc.undelete_file_system(filesystem.name, filesystem.deleted_version,
new_name="restored" + name + str(restored_version))
restored_version += 1
# to make sure the deleted filesystem is restored
props = restored_fs_client.get_file_system_properties()
self.assertIsNotNone(props)
def test_list_system_filesystems(self, datalake_storage_account_name,
datalake_storage_account_key):
self._setUp(datalake_storage_account_name,
datalake_storage_account_key)
# Arrange
dsc = DataLakeServiceClient(self.dsc.url,
credential=datalake_storage_account_key)
# Act
filesystems = list(dsc.list_file_systems(include_system=True))
# Assert
found = False
for fs in filesystems:
if fs.name == "$logs":
found = True
self.assertEqual(found, True)
class FileSystemTest(StorageTestCase):
def setUp(self):
super(FileSystemTest, self).setUp()
url = self._get_account_url()
self.dsc = DataLakeServiceClient(
url, credential=self.settings.STORAGE_DATA_LAKE_ACCOUNT_KEY)
self.config = self.dsc._config
self.test_file_systems = []
def tearDown(self):
if not self.is_playback():
try:
for file_system in self.test_file_systems:
self.dsc.delete_file_system(file_system)
except:
pass
return super(FileSystemTest, self).tearDown()
# --Helpers-----------------------------------------------------------------
def _get_file_system_reference(self, prefix=TEST_FILE_SYSTEM_PREFIX):
file_system_name = self.get_resource_name(prefix)
self.test_file_systems.append(file_system_name)
return file_system_name
def _create_file_system(self, file_system_prefix=TEST_FILE_SYSTEM_PREFIX):
return self.dsc.create_file_system(
self._get_file_system_reference(prefix=file_system_prefix))
# --Helpers-----------------------------------------------------------------
@record
def test_create_file_system(self):
# Arrange
file_system_name = self._get_file_system_reference()
# Act
file_system_client = self.dsc.get_file_system_client(file_system_name)
created = file_system_client.create_file_system()
# Assert
self.assertTrue(created)
@record
def test_create_file_system_with_metadata(self):
# Arrange
metadata = {'hello': 'world', 'number': '42'}
file_system_name = self._get_file_system_reference()
# Act
file_system_client = self.dsc.get_file_system_client(file_system_name)
created = file_system_client.create_file_system(metadata=metadata)
# Assert
meta = file_system_client.get_file_system_properties().metadata
self.assertTrue(created)
self.assertDictEqual(meta, metadata)
@record
def test_set_file_system_acl(self):
# Act
file_system = self._create_file_system()
access_policy = AccessPolicy(
permission=FileSystemSasPermissions(read=True),
expiry=datetime.utcnow() + timedelta(hours=1),
start=datetime.utcnow())
signed_identifier1 = {'testid': access_policy}
response = file_system.set_file_system_access_policy(
signed_identifier1, public_access=PublicAccess.FileSystem)
self.assertIsNotNone(response.get('etag'))
self.assertIsNotNone(response.get('last_modified'))
acl1 = file_system.get_file_system_access_policy()
self.assertIsNotNone(acl1['public_access'])
self.assertEqual(len(acl1['signed_identifiers']), 1)
# If set signed identifier without specifying the access policy then it will be default to None
signed_identifier2 = {'testid': access_policy, 'test2': access_policy}
file_system.set_file_system_access_policy(signed_identifier2)
acl2 = file_system.get_file_system_access_policy()
self.assertIsNone(acl2['public_access'])
self.assertEqual(len(acl2['signed_identifiers']), 2)
@record
def test_list_file_systemss(self):
# Arrange
file_system_name = self._get_file_system_reference()
file_system = self.dsc.create_file_system(file_system_name)
# Act
file_systems = list(self.dsc.list_file_systems())
# Assert
self.assertIsNotNone(file_systems)
self.assertGreaterEqual(len(file_systems), 1)
self.assertIsNotNone(file_systems[0])
self.assertNamedItemInContainer(file_systems,
file_system.file_system_name)
self.assertIsNotNone(file_systems[0].has_immutability_policy)
self.assertIsNotNone(file_systems[0].has_legal_hold)
@record
def test_delete_file_system_with_existing_file_system(self):
# Arrange
file_system = self._create_file_system()
# Act
deleted = file_system.delete_file_system()
# Assert
self.assertIsNone(deleted)
@record
def test_delete_none_existing_file_system(self):
fake_file_system_client = self.dsc.get_file_system_client("fakeclient")
# Act
with self.assertRaises(ResourceNotFoundError):
fake_file_system_client.delete_file_system(
match_condition=MatchConditions.IfMissing)
@record
def test_list_file_systems_with_include_metadata(self):
# Arrange
file_system = self._create_file_system()
metadata = {'hello': 'world', 'number': '42'}
resp = file_system.set_file_system_metadata(metadata)
# Act
file_systems = list(
self.dsc.list_file_systems(
name_starts_with=file_system.file_system_name,
include_metadata=True))
# Assert
self.assertIsNotNone(file_systems)
self.assertGreaterEqual(len(file_systems), 1)
self.assertIsNotNone(file_systems[0])
self.assertNamedItemInContainer(file_systems,
file_system.file_system_name)
self.assertDictEqual(file_systems[0].metadata, metadata)
@record
def test_list_file_systems_by_page(self):
# Arrange
for i in range(0, 6):
self._create_file_system(
file_system_prefix="filesystem{}".format(i))
# Act
file_systems = list(
next(
self.dsc.list_file_systems(results_per_page=3,
name_starts_with="file",
include_metadata=True).by_page()))
# Assert
self.assertIsNotNone(file_systems)
self.assertGreaterEqual(len(file_systems), 3)
@record
def test_list_file_systems_with_public_access(self):
# Arrange
file_system_name = self._get_file_system_reference()
file_system = self.dsc.get_file_system_client(file_system_name)
file_system.create_file_system(public_access="blob")
metadata = {'hello': 'world', 'number': '42'}
resp = file_system.set_file_system_metadata(metadata)
# Act
file_systems = list(
self.dsc.list_file_systems(
name_starts_with=file_system.file_system_name,
include_metadata=True))
# Assert
self.assertIsNotNone(file_systems)
self.assertGreaterEqual(len(file_systems), 1)
self.assertIsNotNone(file_systems[0])
self.assertNamedItemInContainer(file_systems,
file_system.file_system_name)
self.assertDictEqual(file_systems[0].metadata, metadata)
self.assertTrue(file_systems[0].public_access is PublicAccess.File)
@record
def test_get_file_system_properties(self):
# Arrange
metadata = {'hello': 'world', 'number': '42'}
file_system = self._create_file_system()
file_system.set_file_system_metadata(metadata)
# Act
props = file_system.get_file_system_properties()
# Assert
self.assertIsNotNone(props)
self.assertDictEqual(props.metadata, metadata)
self.assertIsNotNone(props.has_immutability_policy)
self.assertIsNotNone(props.has_legal_hold)
@record
def test_list_paths(self):
# Arrange
file_system = self._create_file_system()
for i in range(0, 6):
file_system.create_directory("dir1{}".format(i))
paths = list(file_system.get_paths(upn=True))
self.assertEqual(len(paths), 6)
@record
def test_list_paths_which_are_all_files(self):
# Arrange
file_system = self._create_file_system()
for i in range(0, 6):
file_system.create_file("file{}".format(i))
paths = list(file_system.get_paths(upn=True))
self.assertEqual(len(paths), 6)
@record
def test_list_paths_with_max_per_page(self):
# Arrange
file_system = self._create_file_system()
for i in range(0, 6):
file_system.create_directory("dir1{}".format(i))
generator1 = file_system.get_paths(max_results=2, upn=True).by_page()
paths1 = list(next(generator1))
generator2 = file_system.get_paths(max_results=4, upn=True)\
.by_page(continuation_token=generator1.continuation_token)
paths2 = list(next(generator2))
self.assertEqual(len(paths1), 2)
self.assertEqual(len(paths2), 4)
@record
def test_list_paths_under_specific_path(self):
# Arrange
file_system = self._create_file_system()
for i in range(0, 6):
file_system.create_directory("dir1{}".format(i))
# create a subdirectory under the current directory
subdir = file_system.get_directory_client(
"dir1{}".format(i)).create_sub_directory("subdir")
subdir.create_sub_directory("subsub")
# create a file under the current directory
file_client = subdir.create_file("file")
file_client.append_data(b"abced", 0, 5)
file_client.flush_data(5)
generator1 = file_system.get_paths(path="dir10/subdir",
max_results=2,
upn=True).by_page()
paths = list(next(generator1))
self.assertEqual(len(paths), 2)
self.assertEqual(paths[0].content_length, 5)
@record
def test_list_paths_recursively(self):
# Arrange
file_system = self._create_file_system()
for i in range(0, 6):
file_system.create_directory("dir1{}".format(i))
# create a subdirectory under the current directory
subdir = file_system.get_directory_client(
"dir1{}".format(i)).create_sub_directory("subdir")
subdir.create_sub_directory("subsub")
# create a file under the current directory
subdir.create_file("file")
paths = list(file_system.get_paths(recursive=True, upn=True))
# there are 24 subpaths in total
self.assertEqual(len(paths), 24)
@record
def test_create_directory_from_file_system_client(self):
# Arrange
file_system = self._create_file_system()
file_system.create_directory("dir1/dir2")
paths = list(file_system.get_paths(recursive=False, upn=True))
self.assertEqual(len(paths), 1)
self.assertEqual(paths[0].name, "dir1")
@record
def test_create_file_from_file_system_client(self):
# Arrange
file_system = self._create_file_system()
file_system.create_file("dir1/dir2/file")
paths = list(file_system.get_paths(recursive=True, upn=True))
self.assertEqual(len(paths), 3)
self.assertEqual(paths[0].name, "dir1")
self.assertEqual(paths[2].is_directory, False)
class FileSystemTest(StorageTestCase):
def _setUp(self, account_name, account_key):
url = self._get_account_url(account_name)
self.dsc = DataLakeServiceClient(url, account_key)
self.config = self.dsc._config
self.test_file_systems = []
def tearDown(self):
if not self.is_playback():
try:
for file_system in self.test_file_systems:
self.dsc.delete_file_system(file_system)
except:
pass
return super(FileSystemTest, self).tearDown()
# --Helpers-----------------------------------------------------------------
def _get_file_system_reference(self, prefix=TEST_FILE_SYSTEM_PREFIX):
file_system_name = self.get_resource_name(prefix)
self.test_file_systems.append(file_system_name)
return file_system_name
def _create_file_system(self, file_system_prefix=TEST_FILE_SYSTEM_PREFIX):
return self.dsc.create_file_system(self._get_file_system_reference(prefix=file_system_prefix))
# --Helpers-----------------------------------------------------------------
@DataLakePreparer()
def test_create_file_system(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# Arrange
file_system_name = self._get_file_system_reference()
# Act
file_system_client = self.dsc.get_file_system_client(file_system_name)
created = file_system_client.create_file_system()
# Assert
self.assertTrue(created)
@DataLakePreparer()
def test_file_system_exists(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# Arrange
file_system_name = self._get_file_system_reference()
# Act
file_system_client1 = self.dsc.get_file_system_client(file_system_name)
file_system_client2 = self.dsc.get_file_system_client("nonexistentfs")
file_system_client1.create_file_system()
self.assertTrue(file_system_client1.exists())
self.assertFalse(file_system_client2.exists())
@DataLakePreparer()
def test_create_file_system_with_metadata(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# Arrange
metadata = {'hello': 'world', 'number': '42'}
file_system_name = self._get_file_system_reference()
# Act
file_system_client = self.dsc.get_file_system_client(file_system_name)
created = file_system_client.create_file_system(metadata=metadata)
# Assert
meta = file_system_client.get_file_system_properties().metadata
self.assertTrue(created)
self.assertDictEqual(meta, metadata)
@DataLakePreparer()
def test_set_file_system_acl(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# Act
file_system = self._create_file_system()
access_policy = AccessPolicy(permission=FileSystemSasPermissions(read=True),
expiry=datetime.utcnow() + timedelta(hours=1),
start=datetime.utcnow())
signed_identifier1 = {'testid': access_policy}
response = file_system.set_file_system_access_policy(signed_identifier1, public_access=PublicAccess.FileSystem)
self.assertIsNotNone(response.get('etag'))
self.assertIsNotNone(response.get('last_modified'))
acl1 = file_system.get_file_system_access_policy()
self.assertIsNotNone(acl1['public_access'])
self.assertEqual(len(acl1['signed_identifiers']), 1)
# If set signed identifier without specifying the access policy then it will be default to None
signed_identifier2 = {'testid': access_policy, 'test2': access_policy}
file_system.set_file_system_access_policy(signed_identifier2)
acl2 = file_system.get_file_system_access_policy()
self.assertIsNone(acl2['public_access'])
self.assertEqual(len(acl2['signed_identifiers']), 2)
@DataLakePreparer()
def test_list_file_systemss(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# Arrange
file_system_name = self._get_file_system_reference()
file_system = self.dsc.create_file_system(file_system_name)
# Act
file_systems = list(self.dsc.list_file_systems())
# Assert
self.assertIsNotNone(file_systems)
self.assertGreaterEqual(len(file_systems), 1)
self.assertIsNotNone(file_systems[0])
self.assertNamedItemInContainer(file_systems, file_system.file_system_name)
self.assertIsNotNone(file_systems[0].has_immutability_policy)
self.assertIsNotNone(file_systems[0].has_legal_hold)
@DataLakePreparer()
def test_rename_file_system(self, datalake_storage_account_name, datalake_storage_account_key):
if not self.is_playback():
return
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
old_name1 = self._get_file_system_reference(prefix="oldcontainer1")
old_name2 = self._get_file_system_reference(prefix="oldcontainer2")
new_name = self._get_file_system_reference(prefix="newcontainer")
filesystem1 = self.dsc.create_file_system(old_name1)
self.dsc.create_file_system(old_name2)
new_filesystem = self.dsc._rename_file_system(name=old_name1, new_name=new_name)
with self.assertRaises(HttpResponseError):
self.dsc._rename_file_system(name=old_name2, new_name=new_name)
with self.assertRaises(HttpResponseError):
filesystem1.get_file_system_properties()
with self.assertRaises(HttpResponseError):
self.dsc._rename_file_system(name="badfilesystem", new_name="filesystem")
self.assertEqual(new_name, new_filesystem.get_file_system_properties().name)
@DataLakePreparer()
def test_rename_file_system_with_file_system_client(self, datalake_storage_account_name, datalake_storage_account_key):
pytest.skip("Feature not yet enabled. Make sure to record this test once enabled.")
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
old_name1 = self._get_file_system_reference(prefix="oldcontainer1")
old_name2 = self._get_file_system_reference(prefix="oldcontainer2")
new_name = self._get_file_system_reference(prefix="newcontainer")
bad_name = self._get_file_system_reference(prefix="badcontainer")
filesystem1 = self.dsc.create_file_system(old_name1)
file_system2 = self.dsc.create_file_system(old_name2)
bad_file_system = self.dsc.get_file_system_client(bad_name)
new_filesystem = filesystem1._rename_file_system(new_name=new_name)
with self.assertRaises(HttpResponseError):
file_system2._rename_file_system(new_name=new_name)
with self.assertRaises(HttpResponseError):
filesystem1.get_file_system_properties()
with self.assertRaises(HttpResponseError):
bad_file_system._rename_file_system(new_name="filesystem")
self.assertEqual(new_name, new_filesystem.get_file_system_properties().name)
@DataLakePreparer()
def test_rename_file_system_with_source_lease(self, datalake_storage_account_name, datalake_storage_account_key):
if not self.is_playback():
return
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
old_name = self._get_file_system_reference(prefix="old")
new_name = self._get_file_system_reference(prefix="new")
filesystem = self.dsc.create_file_system(old_name)
filesystem_lease_id = filesystem.acquire_lease()
with self.assertRaises(HttpResponseError):
self.dsc._rename_file_system(name=old_name, new_name=new_name)
with self.assertRaises(HttpResponseError):
self.dsc._rename_file_system(name=old_name, new_name=new_name, lease="bad_id")
new_filesystem = self.dsc._rename_file_system(name=old_name, new_name=new_name, lease=filesystem_lease_id)
self.assertEqual(new_name, new_filesystem.get_file_system_properties().name)
@DataLakePreparer()
def test_undelete_file_system(self, datalake_storage_account_name, datalake_storage_account_key):
# Needs soft delete enabled account.
if not self.is_playback():
return
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
name = self._get_file_system_reference()
filesystem_client = self.dsc.create_file_system(name)
# Act
filesystem_client.delete_file_system()
# to make sure the filesystem deleted
with self.assertRaises(ResourceNotFoundError):
filesystem_client.get_file_system_properties()
filesystem_list = list(self.dsc.list_file_systems(include_deleted=True))
self.assertTrue(len(filesystem_list) >= 1)
restored_version = 0
for filesystem in filesystem_list:
# find the deleted filesystem and restore it
if filesystem.deleted and filesystem.name == filesystem_client.file_system_name:
restored_fs_client = self.dsc.undelete_file_system(filesystem.name, filesystem.deleted_version,
new_name="restored" + name + str(restored_version))
restored_version += 1
# to make sure the deleted filesystem is restored
props = restored_fs_client.get_file_system_properties()
self.assertIsNotNone(props)
@DataLakePreparer()
def test_restore_to_existing_file_system(self, datalake_storage_account_name, datalake_storage_account_key):
# Needs soft delete enabled account.
if not self.is_playback():
return
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# get an existing filesystem
existing_name = self._get_file_system_reference(prefix="existing2")
name = self._get_file_system_reference(prefix="filesystem2")
existing_filesystem_client = self.dsc.create_file_system(existing_name)
filesystem_client = self.dsc.create_file_system(name)
# Act
filesystem_client.delete_file_system()
# to make sure the filesystem deleted
with self.assertRaises(ResourceNotFoundError):
filesystem_client.get_file_system_properties()
filesystem_list = list(self.dsc.list_file_systems(include_deleted=True))
self.assertTrue(len(filesystem_list) >= 1)
for filesystem in filesystem_list:
# find the deleted filesystem and restore it
if filesystem.deleted and filesystem.name == filesystem_client.file_system_name:
with self.assertRaises(HttpResponseError):
self.dsc.undelete_file_system(filesystem.name, filesystem.deleted_version,
new_name=existing_filesystem_client.file_system_name)
@DataLakePreparer()
def test_restore_file_system_with_sas(self, datalake_storage_account_name, datalake_storage_account_key):
pytest.skip(
"We are generating a SAS token therefore play only live but we also need a soft delete enabled account.")
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
token = generate_account_sas(
self.dsc.account_name,
self.dsc.credential.account_key,
ResourceTypes(service=True, file_system=True),
AccountSasPermissions(read=True, write=True, list=True, delete=True),
datetime.utcnow() + timedelta(hours=1),
)
dsc = DataLakeServiceClient(self.dsc.url, token)
name = self._get_file_system_reference(prefix="filesystem")
filesystem_client = dsc.create_file_system(name)
filesystem_client.delete_file_system()
# to make sure the filesystem is deleted
with self.assertRaises(ResourceNotFoundError):
filesystem_client.get_file_system_properties()
filesystem_list = list(dsc.list_file_systems(include_deleted=True))
self.assertTrue(len(filesystem_list) >= 1)
restored_version = 0
for filesystem in filesystem_list:
# find the deleted filesystem and restore it
if filesystem.deleted and filesystem.name == filesystem_client.file_system_name:
restored_fs_client = dsc.undelete_file_system(filesystem.name, filesystem.deleted_version,
new_name="restored" + name + str(restored_version))
restored_version += 1
# to make sure the deleted filesystem is restored
props = restored_fs_client.get_file_system_properties()
self.assertIsNotNone(props)
@DataLakePreparer()
def test_delete_file_system_with_existing_file_system(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# Arrange
file_system = self._create_file_system()
# Act
deleted = file_system.delete_file_system()
# Assert
self.assertIsNone(deleted)
@DataLakePreparer()
def test_delete_none_existing_file_system(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
fake_file_system_client = self.dsc.get_file_system_client("fakeclient")
# Act
with self.assertRaises(ResourceNotFoundError):
fake_file_system_client.delete_file_system(match_condition=MatchConditions.IfMissing)
@DataLakePreparer()
def test_list_file_systems_with_include_metadata(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# Arrange
file_system = self._create_file_system()
metadata = {'hello': 'world', 'number': '42'}
resp = file_system.set_file_system_metadata(metadata)
# Act
file_systems = list(self.dsc.list_file_systems(
name_starts_with=file_system.file_system_name,
include_metadata=True))
# Assert
self.assertIsNotNone(file_systems)
self.assertGreaterEqual(len(file_systems), 1)
self.assertIsNotNone(file_systems[0])
self.assertNamedItemInContainer(file_systems, file_system.file_system_name)
self.assertDictEqual(file_systems[0].metadata, metadata)
@DataLakePreparer()
def test_list_file_systems_by_page(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# Arrange
for i in range(0, 6):
self._create_file_system(file_system_prefix="filesystem{}".format(i))
# Act
file_systems = list(next(self.dsc.list_file_systems(
results_per_page=3,
name_starts_with="file",
include_metadata=True).by_page()))
# Assert
self.assertIsNotNone(file_systems)
self.assertGreaterEqual(len(file_systems), 3)
@DataLakePreparer()
def test_list_file_systems_with_public_access(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# Arrange
file_system_name = self._get_file_system_reference()
file_system = self.dsc.get_file_system_client(file_system_name)
file_system.create_file_system(public_access="blob")
metadata = {'hello': 'world', 'number': '42'}
resp = file_system.set_file_system_metadata(metadata)
# Act
file_systems = list(self.dsc.list_file_systems(
name_starts_with=file_system.file_system_name,
include_metadata=True))
# Assert
self.assertIsNotNone(file_systems)
self.assertGreaterEqual(len(file_systems), 1)
self.assertIsNotNone(file_systems[0])
self.assertNamedItemInContainer(file_systems, file_system.file_system_name)
self.assertDictEqual(file_systems[0].metadata, metadata)
self.assertTrue(file_systems[0].public_access is PublicAccess.File)
@DataLakePreparer()
def test_get_file_system_properties(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# Arrange
metadata = {'hello': 'world', 'number': '42'}
file_system = self._create_file_system()
file_system.set_file_system_metadata(metadata)
# Act
props = file_system.get_file_system_properties()
# Assert
self.assertIsNotNone(props)
self.assertDictEqual(props.metadata, metadata)
self.assertIsNotNone(props.has_immutability_policy)
self.assertIsNotNone(props.has_legal_hold)
@DataLakePreparer()
def test_service_client_session_closes_after_filesystem_creation(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# Arrange
dsc2 = DataLakeServiceClient(self.dsc.url, credential=datalake_storage_account_key)
with DataLakeServiceClient(self.dsc.url, credential=datalake_storage_account_key) as ds_client:
fs1 = ds_client.create_file_system(self._get_file_system_reference(prefix="fs1"))
fs1.delete_file_system()
dsc2.create_file_system(self._get_file_system_reference(prefix="fs2"))
dsc2.close()
@DataLakePreparer()
def test_list_paths(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# Arrange
file_system = self._create_file_system()
for i in range(0, 6):
file_system.create_directory("dir1{}".format(i))
paths = list(file_system.get_paths(upn=True))
self.assertEqual(len(paths), 6)
self.assertTrue(isinstance(paths[0].last_modified, datetime))
@DataLakePreparer()
def test_list_paths_which_are_all_files(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# Arrange
file_system = self._create_file_system()
for i in range(0, 6):
file_system.create_file("file{}".format(i))
paths = list(file_system.get_paths(upn=True))
self.assertEqual(len(paths), 6)
@DataLakePreparer()
def test_list_paths_with_max_per_page(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# Arrange
file_system = self._create_file_system()
for i in range(0, 6):
file_system.create_directory("dir1{}".format(i))
generator1 = file_system.get_paths(max_results=2, upn=True).by_page()
paths1 = list(next(generator1))
generator2 = file_system.get_paths(max_results=4, upn=True)\
.by_page(continuation_token=generator1.continuation_token)
paths2 = list(next(generator2))
self.assertEqual(len(paths1), 2)
self.assertEqual(len(paths2), 4)
@DataLakePreparer()
def test_list_paths_under_specific_path(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# Arrange
file_system = self._create_file_system()
for i in range(0, 6):
file_system.create_directory("dir1{}".format(i))
# create a subdirectory under the current directory
subdir = file_system.get_directory_client("dir1{}".format(i)).create_sub_directory("subdir")
subdir.create_sub_directory("subsub")
# create a file under the current directory
file_client = subdir.create_file("file")
file_client.append_data(b"abced", 0, 5)
file_client.flush_data(5)
generator1 = file_system.get_paths(path="dir10/subdir", max_results=2, upn=True).by_page()
paths = list(next(generator1))
self.assertEqual(len(paths), 2)
self.assertEqual(paths[0].content_length, 5)
@DataLakePreparer()
def test_list_paths_recursively(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# Arrange
file_system = self._create_file_system()
for i in range(0, 6):
file_system.create_directory("dir1{}".format(i))
# create a subdirectory under the current directory
subdir = file_system.get_directory_client("dir1{}".format(i)).create_sub_directory("subdir")
subdir.create_sub_directory("subsub")
# create a file under the current directory
subdir.create_file("file")
paths = list(file_system.get_paths(recursive=True, upn=True))
# there are 24 subpaths in total
self.assertEqual(len(paths), 24)
@DataLakePreparer()
def test_list_paths_pages_correctly(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# Arrange
file_system = self._create_file_system(file_system_prefix="fs1")
for i in range(0, 6):
file_system.create_directory("dir1{}".format(i))
for i in range(0, 6):
file_system.create_file("file{}".format(i))
generator = file_system.get_paths(max_results=6, upn=True).by_page()
paths1 = list(next(generator))
paths2 = list(next(generator))
with self.assertRaises(StopIteration):
list(next(generator))
self.assertEqual(len(paths1), 6)
self.assertEqual(len(paths2), 6)
@DataLakePreparer()
def test_create_directory_from_file_system_client(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# Arrange
file_system = self._create_file_system()
file_system.create_directory("dir1/dir2")
paths = list(file_system.get_paths(recursive=False, upn=True))
self.assertEqual(len(paths), 1)
self.assertEqual(paths[0].name, "dir1")
@DataLakePreparer()
def test_create_file_from_file_system_client(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# Arrange
file_system = self._create_file_system()
file_system.create_file("dir1/dir2/file")
paths = list(file_system.get_paths(recursive=True, upn=True))
self.assertEqual(len(paths), 3)
self.assertEqual(paths[0].name, "dir1")
self.assertEqual(paths[2].is_directory, False)
@DataLakePreparer()
def test_get_root_directory_client(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
file_system = self._create_file_system()
directory_client = file_system._get_root_directory_client()
acl = 'user::rwx,group::r-x,other::rwx'
directory_client.set_access_control(acl=acl)
access_control = directory_client.get_access_control()
self.assertEqual(acl, access_control['acl'])
@DataLakePreparer()
def test_file_system_sessions_closes_properly(self, datalake_storage_account_name, datalake_storage_account_key):
self._setUp(datalake_storage_account_name, datalake_storage_account_key)
# Arrange
file_system_client = self._create_file_system("fenrhxsbfvsdvdsvdsadb")
with file_system_client as fs_client:
with fs_client.get_file_client("file1.txt") as f_client:
f_client.create_file()
with fs_client.get_file_client("file2.txt") as f_client:
f_client.create_file()
with fs_client.get_directory_client("file1") as f_client:
f_client.create_directory()
with fs_client.get_directory_client("file2") as f_client:
f_client.create_directory()
class DirectoryTest(StorageTestCase):
def setUp(self):
super(DirectoryTest, self).setUp()
url = self._get_account_url()
self.dsc = DataLakeServiceClient(
url, credential=self.settings.STORAGE_DATA_LAKE_ACCOUNT_KEY)
self.config = self.dsc._config
self.file_system_name = self.get_resource_name('filesystem')
if not self.is_playback():
file_system = self.dsc.get_file_system_client(
self.file_system_name)
try:
file_system.create_file_system(timeout=5)
except ResourceExistsError:
pass
def tearDown(self):
if not self.is_playback():
try:
self.dsc.delete_file_system(self.file_system_name)
for file_system in self.dsc.list_file_systems():
self.dsc.delete_file_system(file_system.name)
except:
pass
return super(DirectoryTest, self).tearDown()
# --Helpers-----------------------------------------------------------------
def _get_directory_reference(self, prefix=TEST_DIRECTORY_PREFIX):
directory_name = self.get_resource_name(prefix)
return directory_name
def _create_directory_and_get_directory_client(self, directory_name=None):
directory_name = directory_name if directory_name else self._get_directory_reference(
)
directory_client = self.dsc.get_directory_client(
self.file_system_name, directory_name)
directory_client.create_directory()
return directory_client
def _create_file_system(self):
return self.dsc.create_file_system(self._get_file_system_reference())
# --Helpers-----------------------------------------------------------------
@record
def test_create_directory(self):
# Arrange
directory_name = self._get_directory_reference()
content_settings = ContentSettings(content_language='spanish',
content_disposition='inline')
# Act
directory_client = self.dsc.get_directory_client(
self.file_system_name, directory_name)
created = directory_client.create_directory(
content_settings=content_settings)
# Assert
self.assertTrue(created)
@record
def test_using_oauth_token_credential_to_create_directory(self):
# generate a token with directory level create permission
directory_name = self._get_directory_reference()
token_credential = self.generate_oauth_token()
directory_client = DataLakeDirectoryClient(self.dsc.url,
self.file_system_name,
directory_name,
credential=token_credential)
response = directory_client.create_directory()
self.assertIsNotNone(response)
@record
def test_create_directory_with_match_conditions(self):
# Arrange
directory_name = self._get_directory_reference()
# Act
directory_client = self.dsc.get_directory_client(
self.file_system_name, directory_name)
created = directory_client.create_directory(
match_condition=MatchConditions.IfMissing)
# Assert
self.assertTrue(created)
@record
def test_create_directory_with_permission(self):
# Arrange
directory_name = self._get_directory_reference()
# Act
directory_client = self.dsc.get_directory_client(
self.file_system_name, directory_name)
created = directory_client.create_directory(permissions="rwxr--r--",
umask="0000")
prop = directory_client.get_access_control()
# Assert
self.assertTrue(created)
self.assertEqual(prop['permissions'], 'rwxr--r--')
@record
def test_create_directory_with_content_settings(self):
# Arrange
directory_name = self._get_directory_reference()
content_settings = ContentSettings(content_language='spanish',
content_disposition='inline')
# Act
directory_client = self.dsc.get_directory_client(
self.file_system_name, directory_name)
created = directory_client.create_directory(
content_settings=content_settings)
# Assert
self.assertTrue(created)
@record
def test_create_directory_with_metadata(self):
# Arrange
directory_name = self._get_directory_reference()
metadata = {'hello': 'world', 'number': '42'}
# Act
directory_client = self.dsc.get_directory_client(
self.file_system_name, directory_name)
created = directory_client.create_directory(metadata=metadata)
properties = directory_client.get_directory_properties()
# Assert
self.assertTrue(created)
@record
def test_delete_directory(self):
# Arrange
directory_name = self._get_directory_reference()
metadata = {'hello': 'world', 'number': '42'}
directory_client = self.dsc.get_directory_client(
self.file_system_name, directory_name)
directory_client.create_directory(metadata=metadata)
response = directory_client.delete_directory()
# Assert
self.assertIsNone(response)
@record
def test_delete_directory_with_if_modified_since(self):
# Arrange
directory_name = self._get_directory_reference()
directory_client = self.dsc.get_directory_client(
self.file_system_name, directory_name)
directory_client.create_directory()
prop = directory_client.get_directory_properties()
with self.assertRaises(ResourceModifiedError):
directory_client.delete_directory(
if_modified_since=prop['last_modified'])
@record
def test_create_sub_directory_and_delete_sub_directory(self):
# Arrange
directory_name = self._get_directory_reference()
metadata = {'hello': 'world', 'number': '42'}
# Create a directory first, to prepare for creating sub directory
directory_client = self.dsc.get_directory_client(
self.file_system_name, directory_name)
directory_client.create_directory(metadata=metadata)
# Create sub directory from the current directory
sub_directory_name = 'subdir'
sub_directory_created = directory_client.create_sub_directory(
sub_directory_name)
# to make sure the sub directory was indeed created by get sub_directory properties from sub directory client
sub_directory_client = self.dsc.get_directory_client(
self.file_system_name, directory_name + '/' + sub_directory_name)
sub_properties = sub_directory_client.get_directory_properties()
# Assert
self.assertTrue(sub_directory_created)
self.assertTrue(sub_properties)
# Act
directory_client.delete_sub_directory(sub_directory_name)
with self.assertRaises(ResourceNotFoundError):
sub_directory_client.get_directory_properties()
@record
def test_set_access_control(self):
directory_name = self._get_directory_reference()
metadata = {'hello': 'world', 'number': '42'}
directory_client = self.dsc.get_directory_client(
self.file_system_name, directory_name)
directory_client.create_directory(metadata=metadata)
response = directory_client.set_access_control(permissions='0777')
# Assert
self.assertIsNotNone(response)
@record
def test_set_access_control_with_acl(self):
directory_name = self._get_directory_reference()
metadata = {'hello': 'world', 'number': '42'}
directory_client = self.dsc.get_directory_client(
self.file_system_name, directory_name)
directory_client.create_directory(metadata=metadata)
acl = 'user::rwx,group::r-x,other::rwx'
directory_client.set_access_control(acl=acl)
access_control = directory_client.get_access_control()
# Assert
self.assertIsNotNone(access_control)
self.assertEqual(acl, access_control['acl'])
@record
def test_set_access_control_if_none_modified(self):
directory_name = self._get_directory_reference()
directory_client = self.dsc.get_directory_client(
self.file_system_name, directory_name)
resp = directory_client.create_directory()
response = directory_client.set_access_control(
permissions='0777',
etag=resp['etag'],
match_condition=MatchConditions.IfNotModified)
# Assert
self.assertIsNotNone(response)
@record
def test_get_access_control(self):
directory_name = self._get_directory_reference()
metadata = {'hello': 'world', 'number': '42'}
directory_client = self.dsc.get_directory_client(
self.file_system_name, directory_name)
directory_client.create_directory(metadata=metadata,
permissions='0777')
# Act
response = directory_client.get_access_control()
# Assert
self.assertIsNotNone(response)
@record
def test_get_access_control_with_match_conditions(self):
directory_name = self._get_directory_reference()
directory_client = self.dsc.get_directory_client(
self.file_system_name, directory_name)
resp = directory_client.create_directory(permissions='0777',
umask='0000')
# Act
response = directory_client.get_access_control(
etag=resp['etag'], match_condition=MatchConditions.IfNotModified)
# Assert
self.assertIsNotNone(response)
self.assertEquals(response['permissions'], 'rwxrwxrwx')
@record
def test_rename_from(self):
metadata = {'hello': 'world', 'number': '42'}
directory_name = self._get_directory_reference()
directory_client = self.dsc.get_directory_client(
self.file_system_name, directory_name)
directory_client.create_directory()
new_name = "newname"
new_directory_client = self.dsc.get_directory_client(
self.file_system_name, new_name)
new_directory_client._rename_path('/' + self.file_system_name + '/' +
directory_name,
metadata=metadata)
properties = new_directory_client.get_directory_properties()
self.assertIsNotNone(properties)
@record
def test_rename_from_a_shorter_directory_to_longer_directory(self):
# TODO: investigate why rename shorter path to a longer one does not work
pytest.skip("")
directory_name = self._get_directory_reference()
self._create_directory_and_get_directory_client(directory_name="old")
new_name = "newname"
new_directory_client = self._create_directory_and_get_directory_client(
directory_name=new_name)
new_directory_client = new_directory_client.create_sub_directory(
"newsub")
new_directory_client._rename_path('/' + self.file_system_name + '/' +
directory_name)
properties = new_directory_client.get_directory_properties()
self.assertIsNotNone(properties)
@record
def test_rename_from_a_directory_in_another_file_system(self):
# create a file dir1 under file system1
old_file_system_name = "oldfilesystem"
old_dir_name = "olddir"
old_client = self.dsc.get_file_system_client(old_file_system_name)
old_client.create_file_system()
old_client.create_directory(old_dir_name)
# create a dir2 under file system2
new_name = "newname"
new_directory_client = self._create_directory_and_get_directory_client(
directory_name=new_name)
new_directory_client = new_directory_client.create_sub_directory(
"newsub")
# rename dir1 under file system1 to dir2 under file system2
new_directory_client._rename_path('/' + old_file_system_name + '/' +
old_dir_name)
properties = new_directory_client.get_directory_properties()
self.assertIsNotNone(properties)
self.dsc.delete_file_system(old_file_system_name)
@record
def test_rename_to_an_existing_directory_in_another_file_system(self):
# create a file dir1 under file system1
destination_file_system_name = "destfilesystem"
destination_dir_name = "destdir"
fs_client = self.dsc.get_file_system_client(
destination_file_system_name)
fs_client.create_file_system()
destination_directory_client = fs_client.create_directory(
destination_dir_name)
# create a dir2 under file system2
source_name = "source"
source_directory_client = self._create_directory_and_get_directory_client(
directory_name=source_name)
source_directory_client = source_directory_client.create_sub_directory(
"subdir")
# rename dir2 under file system2 to dir1 under file system1
res = source_directory_client.rename_directory(
'/' + destination_file_system_name + '/' + destination_dir_name)
# the source directory has been renamed to destination directory, so it cannot be found
with self.assertRaises(HttpResponseError):
source_directory_client.get_directory_properties()
self.assertEquals(res.url, destination_directory_client.url)
@record
def test_rename_with_none_existing_destination_condition_and_source_unmodified_condition(
self):
non_existing_dir_name = "nonexistingdir"
# create a file system1
destination_file_system_name = self._get_directory_reference(
"destfilesystem")
fs_client = self.dsc.get_file_system_client(
destination_file_system_name)
fs_client.create_file_system()
# create a dir2 under file system2
source_name = "source"
source_directory_client = self._create_directory_and_get_directory_client(
directory_name=source_name)
source_directory_client = source_directory_client.create_sub_directory(
"subdir")
# rename dir2 under file system2 to a non existing directory under file system1,
# when dir1 does not exist and dir2 wasn't modified
etag = source_directory_client.get_directory_properties()['etag']
res = source_directory_client.rename_directory(
'/' + destination_file_system_name + '/' + non_existing_dir_name,
match_condition=MatchConditions.IfMissing,
source_etag=etag,
source_match_condition=MatchConditions.IfNotModified)
# the source directory has been renamed to destination directory, so it cannot be found
with self.assertRaises(HttpResponseError):
source_directory_client.get_directory_properties()
self.assertEquals(non_existing_dir_name, res.path_name)
@record
def test_rename_to_an_non_existing_directory_in_another_file_system(self):
# create a file dir1 under file system1
destination_file_system_name = self._get_directory_reference(
"destfilesystem")
non_existing_dir_name = "nonexistingdir"
fs_client = self.dsc.get_file_system_client(
destination_file_system_name)
fs_client.create_file_system()
# create a dir2 under file system2
source_name = "source"
source_directory_client = self._create_directory_and_get_directory_client(
directory_name=source_name)
source_directory_client = source_directory_client.create_sub_directory(
"subdir")
# rename dir2 under file system2 to dir1 under file system1
res = source_directory_client.rename_directory(
'/' + destination_file_system_name + '/' + non_existing_dir_name)
# the source directory has been renamed to destination directory, so it cannot be found
with self.assertRaises(HttpResponseError):
source_directory_client.get_directory_properties()
self.assertEquals(non_existing_dir_name, res.path_name)
@record
def test_rename_directory_to_non_empty_directory(self):
# TODO: investigate why rename non empty dir doesn't work
pytest.skip("")
dir1 = self._create_directory_and_get_directory_client("dir1")
dir1.create_sub_directory("subdir")
dir2 = self._create_directory_and_get_directory_client("dir2")
dir2.rename_directory(dir1.file_system_name + '/' + dir1.path_name)
with self.assertRaises(HttpResponseError):
dir2.get_directory_properties()
@record
def test_get_properties(self):
# Arrange
directory_name = self._get_directory_reference()
metadata = {'hello': 'world', 'number': '42'}
directory_client = self.dsc.get_directory_client(
self.file_system_name, directory_name)
directory_client.create_directory(metadata=metadata)
properties = directory_client.get_directory_properties()
# Assert
self.assertTrue(properties)
self.assertIsNotNone(properties.metadata)
self.assertEqual(properties.metadata['hello'], metadata['hello'])
@record
def test_using_directory_sas_to_read(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
client = self._create_directory_and_get_directory_client()
directory_name = client.path_name
# generate a token with directory level read permission
token = generate_directory_sas(
self.dsc.account_name,
self.file_system_name,
directory_name,
account_key=self.dsc.credential.account_key,
permission=DirectorySasPermissions(read=True),
expiry=datetime.utcnow() + timedelta(hours=1),
)
directory_client = DataLakeDirectoryClient(self.dsc.url,
self.file_system_name,
directory_name,
credential=token)
access_control = directory_client.get_access_control()
self.assertIsNotNone(access_control)
@record
def test_using_directory_sas_to_create(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# generate a token with directory level create permission
directory_name = self._get_directory_reference()
token = generate_directory_sas(
self.dsc.account_name,
self.file_system_name,
directory_name,
account_key=self.dsc.credential.account_key,
permission=DirectorySasPermissions(create=True),
expiry=datetime.utcnow() + timedelta(hours=1),
)
directory_client = DataLakeDirectoryClient(self.dsc.url,
self.file_system_name,
directory_name,
credential=token)
response = directory_client.create_directory()
self.assertIsNotNone(response)
def data_lake_service_sample(self):
# Instantiate a DataLakeServiceClient using a connection string
# [START create_datalake_service_client]
from azure.storage.filedatalake import DataLakeServiceClient
datalake_service_client = DataLakeServiceClient.from_connection_string(
self.connection_string)
# [END create_datalake_service_client]
# Instantiate a DataLakeServiceClient Azure Identity credentials.
# [START create_datalake_service_client_oauth]
from azure.identity import ClientSecretCredential
token_credential = ClientSecretCredential(
self.active_directory_tenant_id,
self.active_directory_application_id,
self.active_directory_application_secret,
)
datalake_service_client = DataLakeServiceClient(
"https://{}.dfs.core.windows.net".format(self.account_name),
credential=token_credential)
# [END create_datalake_service_client_oauth]
# get user delegation key
# [START get_user_delegation_key]
from datetime import datetime, timedelta
user_delegation_key = datalake_service_client.get_user_delegation_key(
datetime.utcnow(),
datetime.utcnow() + timedelta(hours=1))
# [END get_user_delegation_key]
# Create file systems
# [START create_file_system_from_service_client]
datalake_service_client.create_file_system("filesystem")
# [END create_file_system_from_service_client]
file_system_client = datalake_service_client.create_file_system(
"anotherfilesystem")
# List file systems
# [START list_file_systems]
file_systems = datalake_service_client.list_file_systems()
for file_system in file_systems:
print(file_system.name)
# [END list_file_systems]
# Get Clients from DataLakeServiceClient
file_system_client = datalake_service_client.get_file_system_client(
file_system_client.file_system_name)
# [START get_directory_client_from_service_client]
directory_client = datalake_service_client.get_directory_client(
file_system_client.file_system_name, "mydirectory")
# [END get_directory_client_from_service_client]
# [START get_file_client_from_service_client]
file_client = datalake_service_client.get_file_client(
file_system_client.file_system_name, "myfile")
# [END get_file_client_from_service_client]
# Create file and set properties
metadata = {'hello': 'world', 'number': '42'}
from azure.storage.filedatalake import ContentSettings
content_settings = ContentSettings(content_language='spanish',
content_disposition='inline')
file_client.create_file(content_settings=content_settings)
file_client.set_metadata(metadata=metadata)
file_props = file_client.get_file_properties()
print(file_props.metadata)
# Create file/directory and set properties
directory_client.create_directory(content_settings=content_settings,
metadata=metadata)
dir_props = directory_client.get_directory_properties()
print(dir_props.metadata)
# Delete File Systems
# [START delete_file_system_from_service_client]
datalake_service_client.delete_file_system("filesystem")
# [END delete_file_system_from_service_client]
file_system_client.delete_file_system()
# In no event shall Microsoft, its authors, owners of this repository or anyone else involved in the creation, production, or delivery of the scripts be liable for any damages whatsoever (including,
# without limitation, damages for loss of business profits, business interruption, loss of business information, or other pecuniary loss) arising out of the use of or inability to use the sample scripts or documentation, even if Microsoft has been advised of the possibility of such damages
#-------------------------------------------------------------------------
#IMPORT THE LIBRARIES INTO YOUR FILE
from azure.storage.filedatalake import DataLakeServiceClient
from azure.storage.filedatalake._shared.base_client import create_configuration
#OPTION 1 - MAKING USE OF CONNECTION STRING AND CREATING THE DATALAKE CLIENT
connection_string = "PUT CONNECTION STRING HERE"
#CREATE THE DATALAKE SERVICE CLIENT
service_client = DataLakeServiceClient.from_connection_string(
connection_string)
##OPTION 2 - MAKING USE OF ACCESS KEY AND CREATING THE DATALAKE CLIENT
storage_account_key = "ACCESS KEY"
storage_account_name = "ACCOUNT NAME"
#CREATE THE DATALAKE SERVICE CLIENT
service_client = DataLakeServiceClient(
account_url="{}://{}.dfs.core.windows.net".format("https",
storage_account_name),
credential=storage_account_key)
#PERFORM THE LISTING OPERATION
file_systems = service_client.list_file_systems()
for file_system in file_systems:
print(file_system.name)
class Adl(object):
def __init__(self):
url = self.datalake_account_url()
key = self.datalake_account_key()
print('url: {}'.format(url))
print('key: {}'.format(key))
self.service_client = DataLakeServiceClient(account_url=url,
credential=key)
print(self.service_client)
def create_fs(self, fsname):
try:
fs_client = self.service_client.get_file_system_client(fsname)
fs_client.create_file_system()
print('create_fs: {}'.format(fsname))
except ResourceExistsError:
pass
def delete_fs(self, fsname):
try:
fs_client = self.service_client.get_file_system_client(fsname)
fs_client.delete_file_system()
except ResourceNotFoundError:
pass
def filesystem_list(self):
return self.service_client.list_file_systems()
def create_dir(self, fsname, dirname):
try:
fs_client = self.service_client.get_file_system_client(fsname)
fs_client.create_directory(dirname)
print('create_dir: {} in fs: {}'.format(dirname, fsname))
except:
pass
def file_list(self, fsname, dirname):
try:
fsc = self.service_client.get_file_system_client(
file_system=fsname)
return fsc.get_paths(path=dirname)
except Exception as e:
return list()
def directory_client(self, fsname, dirname):
try:
fs_client = self.service_client.get_file_system_client(fsname)
return fs_client.get_directory_client(dirname)
except:
return None
def upload_file(self, dir_client, local_path, remote_name, opts={}):
file_client = dir_client.create_file(remote_name)
local_file = open(local_path, 'r')
file_contents = local_file.read()
print('upload_file, opts: {}'.format(opts))
# https://docs.microsoft.com/en-us/python/api/azure-storage-file-datalake/azure.storage.filedatalake.contentsettings?view=azure-python
cs = ContentSettings(**opts)
file_client.upload_data(file_contents,
overwrite=True,
content_settings=cs)
print('upload_file; {} -> {}'.format(local_path, remote_name))
def download_file(self, dir_client, remote_name, local_path):
file_client = dir_client.get_file_client(remote_name)
download = file_client.download_file()
downloaded_bytes = download.readall()
local_file = open(local_path, 'wb')
local_file.write(downloaded_bytes)
local_file.close()
print('download_file; {} -> {}'.format(remote_name, local_path))
def datalake_account_url(self):
storage_account_name = os.environ['AZURE_ADL_ACCOUNT']
return "{}://{}.dfs.core.windows.net".format("https",
storage_account_name)
def datalake_account_key(self):
return os.environ['AZURE_ADL_KEY']
def datalake_account_conn_string(self):
return os.environ['AZURE_ADL_CONNECTION_STRING']
class FileSystemTest(StorageTestCase):
def setUp(self):
super(FileSystemTest, self).setUp()
url = self._get_account_url()
self.dsc = DataLakeServiceClient(url, credential=self.settings.STORAGE_DATA_LAKE_ACCOUNT_KEY)
self.config = self.dsc._config
self.test_file_systems = []
def tearDown(self):
if not self.is_playback():
try:
for file_system in self.test_file_systems:
self.dsc.delete_file_system(file_system)
except:
pass
return super(FileSystemTest, self).tearDown()
# --Helpers-----------------------------------------------------------------
def _get_file_system_reference(self, prefix=TEST_FILE_SYSTEM_PREFIX):
file_system_name = self.get_resource_name(prefix)
self.test_file_systems.append(file_system_name)
return file_system_name
def _create_file_system(self, file_system_prefix=TEST_FILE_SYSTEM_PREFIX):
return self.dsc.create_file_system(self._get_file_system_reference(prefix=file_system_prefix))
# --Helpers-----------------------------------------------------------------
@record
def test_create_file_system(self):
# Arrange
file_system_name = self._get_file_system_reference()
# Act
file_system_client = self.dsc.get_file_system_client(file_system_name)
created = file_system_client.create_file_system()
# Assert
self.assertTrue(created)
@record
def test_list_file_systemss(self):
# Arrange
file_system_name = self._get_file_system_reference()
file_system = self.dsc.create_file_system(file_system_name)
# Act
file_systems = list(self.dsc.list_file_systems())
# Assert
self.assertIsNotNone(file_systems)
self.assertGreaterEqual(len(file_systems), 1)
self.assertIsNotNone(file_systems[0])
self.assertNamedItemInContainer(file_systems, file_system.file_system_name)
self.assertIsNotNone(file_systems[0].has_immutability_policy)
self.assertIsNotNone(file_systems[0].has_legal_hold)
@record
def test_delete_file_system_with_existing_file_system(self):
# Arrange
file_system = self._create_file_system()
# Act
deleted = file_system.delete_file_system()
# Assert
self.assertIsNone(deleted)
@record
def test_list_file_systems_with_include_metadata(self):
# Arrange
file_system = self._create_file_system()
metadata = {'hello': 'world', 'number': '42'}
resp = file_system.set_file_system_metadata(metadata)
# Act
file_systems = list(self.dsc.list_file_systems(
name_starts_with=file_system.file_system_name,
include_metadata=True))
# Assert
self.assertIsNotNone(file_systems)
self.assertGreaterEqual(len(file_systems), 1)
self.assertIsNotNone(file_systems[0])
self.assertNamedItemInContainer(file_systems, file_system.file_system_name)
self.assertDictEqual(file_systems[0].metadata, metadata)
@record
def test_list_file_systems_by_page(self):
# Arrange
for i in range(0, 6):
self._create_file_system(file_system_prefix="filesystem{}".format(i))
# Act
file_systems = list(next(self.dsc.list_file_systems(
results_per_page=3,
name_starts_with="file",
include_metadata=True).by_page()))
# Assert
self.assertIsNotNone(file_systems)
self.assertGreaterEqual(len(file_systems), 3)
@record
def test_list_file_systems_with_public_access(self):
# Arrange
file_system_name = self._get_file_system_reference()
file_system = self.dsc.get_file_system_client(file_system_name)
file_system.create_file_system(public_access="blob")
metadata = {'hello': 'world', 'number': '42'}
resp = file_system.set_file_system_metadata(metadata)
# Act
file_systems = list(self.dsc.list_file_systems(
name_starts_with=file_system.file_system_name,
include_metadata=True))
# Assert
self.assertIsNotNone(file_systems)
self.assertGreaterEqual(len(file_systems), 1)
self.assertIsNotNone(file_systems[0])
self.assertNamedItemInContainer(file_systems, file_system.file_system_name)
self.assertDictEqual(file_systems[0].metadata, metadata)
self.assertTrue(file_systems[0].public_access is PublicAccess.File)
@record
def test_get_file_system_properties(self):
# Arrange
metadata = {'hello': 'world', 'number': '42'}
file_system = self._create_file_system()
file_system.set_file_system_metadata(metadata)
# Act
props = file_system.get_file_system_properties()
# Assert
self.assertIsNotNone(props)
self.assertDictEqual(props.metadata, metadata)
self.assertIsNotNone(props.has_immutability_policy)
self.assertIsNotNone(props.has_legal_hold)
@record
def test_list_paths(self):
# Arrange
file_system = self._create_file_system()
for i in range(0, 6):
file_system.create_directory("dir1{}".format(i))
paths = list(file_system.get_paths(upn=True))
self.assertEqual(len(paths), 6)
@record
def test_list_paths_with_max_per_page(self):
# Arrange
file_system = self._create_file_system()
for i in range(0, 6):
file_system.create_directory("dir1{}".format(i))
generator1 = file_system.get_paths(max_results=2, upn=True).by_page()
paths1 = list(next(generator1))
generator2 = file_system.get_paths(max_results=4, upn=True)\
.by_page(continuation_token=generator1.continuation_token)
paths2 = list(next(generator2))
self.assertEqual(len(paths1), 2)
self.assertEqual(len(paths2), 4)
@record
def test_list_paths_under_specific_path(self):
# Arrange
file_system = self._create_file_system()
for i in range(0, 6):
file_system.create_directory("dir1{}".format(i))
# create a subdirectory under the current directory
subdir = file_system.get_directory_client("dir1{}".format(i)).create_sub_directory("subdir")
subdir.create_sub_directory("subsub")
# create a file under the current directory
file_client = subdir.create_file("file")
file_client.append_data(b"abced", 0, 5)
file_client.flush_data(5)
generator1 = file_system.get_paths(path="dir10/subdir", max_results=2, upn=True).by_page()
paths = list(next(generator1))
self.assertEqual(len(paths), 2)
self.assertEqual(paths[0].content_length, 5)
