def test_restore_file_system_with_sas(self, datalake_storage_account_name, datalake_storage_account_key): pytest.skip( "We are generating a SAS token therefore play only live but we also need a soft delete enabled account.") self._setUp(datalake_storage_account_name, datalake_storage_account_key) token = generate_account_sas( self.dsc.account_name, self.dsc.credential.account_key, ResourceTypes(service=True, file_system=True), AccountSasPermissions(read=True, write=True, list=True, delete=True), datetime.utcnow() + timedelta(hours=1), ) dsc = DataLakeServiceClient(self.dsc.url, token) name = self._get_file_system_reference(prefix="filesystem") filesystem_client = dsc.create_file_system(name) filesystem_client.delete_file_system() # to make sure the filesystem is deleted with self.assertRaises(ResourceNotFoundError): filesystem_client.get_file_system_properties() filesystem_list = list(dsc.list_file_systems(include_deleted=True)) self.assertTrue(len(filesystem_list) >= 1) restored_version = 0 for filesystem in filesystem_list: # find the deleted filesystem and restore it if filesystem.deleted and filesystem.name == filesystem_client.file_system_name: restored_fs_client = dsc.undelete_file_system(filesystem.name, filesystem.deleted_version, new_name="restored" + name + str(restored_version)) restored_version += 1 # to make sure the deleted filesystem is restored props = restored_fs_client.get_file_system_properties() self.assertIsNotNone(props)
def test_list_system_filesystems(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange dsc = DataLakeServiceClient(self.dsc.url, credential=datalake_storage_account_key) # Act filesystems = list(dsc.list_file_systems(include_system=True)) # Assert found = False for fs in filesystems: if fs.name == "$logs": found = True self.assertEqual(found, True)
class FileSystemTest(StorageTestCase): def setUp(self): super(FileSystemTest, self).setUp() url = self._get_account_url() self.dsc = DataLakeServiceClient( url, credential=self.settings.STORAGE_DATA_LAKE_ACCOUNT_KEY) self.config = self.dsc._config self.test_file_systems = [] def tearDown(self): if not self.is_playback(): try: for file_system in self.test_file_systems: self.dsc.delete_file_system(file_system) except: pass return super(FileSystemTest, self).tearDown() # --Helpers----------------------------------------------------------------- def _get_file_system_reference(self, prefix=TEST_FILE_SYSTEM_PREFIX): file_system_name = self.get_resource_name(prefix) self.test_file_systems.append(file_system_name) return file_system_name def _create_file_system(self, file_system_prefix=TEST_FILE_SYSTEM_PREFIX): return self.dsc.create_file_system( self._get_file_system_reference(prefix=file_system_prefix)) # --Helpers----------------------------------------------------------------- @record def test_create_file_system(self): # Arrange file_system_name = self._get_file_system_reference() # Act file_system_client = self.dsc.get_file_system_client(file_system_name) created = file_system_client.create_file_system() # Assert self.assertTrue(created) @record def test_create_file_system_with_metadata(self): # Arrange metadata = {'hello': 'world', 'number': '42'} file_system_name = self._get_file_system_reference() # Act file_system_client = self.dsc.get_file_system_client(file_system_name) created = file_system_client.create_file_system(metadata=metadata) # Assert meta = file_system_client.get_file_system_properties().metadata self.assertTrue(created) self.assertDictEqual(meta, metadata) @record def test_set_file_system_acl(self): # Act file_system = self._create_file_system() access_policy = AccessPolicy( permission=FileSystemSasPermissions(read=True), expiry=datetime.utcnow() + timedelta(hours=1), start=datetime.utcnow()) signed_identifier1 = {'testid': access_policy} response = file_system.set_file_system_access_policy( signed_identifier1, public_access=PublicAccess.FileSystem) self.assertIsNotNone(response.get('etag')) self.assertIsNotNone(response.get('last_modified')) acl1 = file_system.get_file_system_access_policy() self.assertIsNotNone(acl1['public_access']) self.assertEqual(len(acl1['signed_identifiers']), 1) # If set signed identifier without specifying the access policy then it will be default to None signed_identifier2 = {'testid': access_policy, 'test2': access_policy} file_system.set_file_system_access_policy(signed_identifier2) acl2 = file_system.get_file_system_access_policy() self.assertIsNone(acl2['public_access']) self.assertEqual(len(acl2['signed_identifiers']), 2) @record def test_list_file_systemss(self): # Arrange file_system_name = self._get_file_system_reference() file_system = self.dsc.create_file_system(file_system_name) # Act file_systems = list(self.dsc.list_file_systems()) # Assert self.assertIsNotNone(file_systems) self.assertGreaterEqual(len(file_systems), 1) self.assertIsNotNone(file_systems[0]) self.assertNamedItemInContainer(file_systems, file_system.file_system_name) self.assertIsNotNone(file_systems[0].has_immutability_policy) self.assertIsNotNone(file_systems[0].has_legal_hold) @record def test_delete_file_system_with_existing_file_system(self): # Arrange file_system = self._create_file_system() # Act deleted = file_system.delete_file_system() # Assert self.assertIsNone(deleted) @record def test_delete_none_existing_file_system(self): fake_file_system_client = self.dsc.get_file_system_client("fakeclient") # Act with self.assertRaises(ResourceNotFoundError): fake_file_system_client.delete_file_system( match_condition=MatchConditions.IfMissing) @record def test_list_file_systems_with_include_metadata(self): # Arrange file_system = self._create_file_system() metadata = {'hello': 'world', 'number': '42'} resp = file_system.set_file_system_metadata(metadata) # Act file_systems = list( self.dsc.list_file_systems( name_starts_with=file_system.file_system_name, include_metadata=True)) # Assert self.assertIsNotNone(file_systems) self.assertGreaterEqual(len(file_systems), 1) self.assertIsNotNone(file_systems[0]) self.assertNamedItemInContainer(file_systems, file_system.file_system_name) self.assertDictEqual(file_systems[0].metadata, metadata) @record def test_list_file_systems_by_page(self): # Arrange for i in range(0, 6): self._create_file_system( file_system_prefix="filesystem{}".format(i)) # Act file_systems = list( next( self.dsc.list_file_systems(results_per_page=3, name_starts_with="file", include_metadata=True).by_page())) # Assert self.assertIsNotNone(file_systems) self.assertGreaterEqual(len(file_systems), 3) @record def test_list_file_systems_with_public_access(self): # Arrange file_system_name = self._get_file_system_reference() file_system = self.dsc.get_file_system_client(file_system_name) file_system.create_file_system(public_access="blob") metadata = {'hello': 'world', 'number': '42'} resp = file_system.set_file_system_metadata(metadata) # Act file_systems = list( self.dsc.list_file_systems( name_starts_with=file_system.file_system_name, include_metadata=True)) # Assert self.assertIsNotNone(file_systems) self.assertGreaterEqual(len(file_systems), 1) self.assertIsNotNone(file_systems[0]) self.assertNamedItemInContainer(file_systems, file_system.file_system_name) self.assertDictEqual(file_systems[0].metadata, metadata) self.assertTrue(file_systems[0].public_access is PublicAccess.File) @record def test_get_file_system_properties(self): # Arrange metadata = {'hello': 'world', 'number': '42'} file_system = self._create_file_system() file_system.set_file_system_metadata(metadata) # Act props = file_system.get_file_system_properties() # Assert self.assertIsNotNone(props) self.assertDictEqual(props.metadata, metadata) self.assertIsNotNone(props.has_immutability_policy) self.assertIsNotNone(props.has_legal_hold) @record def test_list_paths(self): # Arrange file_system = self._create_file_system() for i in range(0, 6): file_system.create_directory("dir1{}".format(i)) paths = list(file_system.get_paths(upn=True)) self.assertEqual(len(paths), 6) @record def test_list_paths_which_are_all_files(self): # Arrange file_system = self._create_file_system() for i in range(0, 6): file_system.create_file("file{}".format(i)) paths = list(file_system.get_paths(upn=True)) self.assertEqual(len(paths), 6) @record def test_list_paths_with_max_per_page(self): # Arrange file_system = self._create_file_system() for i in range(0, 6): file_system.create_directory("dir1{}".format(i)) generator1 = file_system.get_paths(max_results=2, upn=True).by_page() paths1 = list(next(generator1)) generator2 = file_system.get_paths(max_results=4, upn=True)\ .by_page(continuation_token=generator1.continuation_token) paths2 = list(next(generator2)) self.assertEqual(len(paths1), 2) self.assertEqual(len(paths2), 4) @record def test_list_paths_under_specific_path(self): # Arrange file_system = self._create_file_system() for i in range(0, 6): file_system.create_directory("dir1{}".format(i)) # create a subdirectory under the current directory subdir = file_system.get_directory_client( "dir1{}".format(i)).create_sub_directory("subdir") subdir.create_sub_directory("subsub") # create a file under the current directory file_client = subdir.create_file("file") file_client.append_data(b"abced", 0, 5) file_client.flush_data(5) generator1 = file_system.get_paths(path="dir10/subdir", max_results=2, upn=True).by_page() paths = list(next(generator1)) self.assertEqual(len(paths), 2) self.assertEqual(paths[0].content_length, 5) @record def test_list_paths_recursively(self): # Arrange file_system = self._create_file_system() for i in range(0, 6): file_system.create_directory("dir1{}".format(i)) # create a subdirectory under the current directory subdir = file_system.get_directory_client( "dir1{}".format(i)).create_sub_directory("subdir") subdir.create_sub_directory("subsub") # create a file under the current directory subdir.create_file("file") paths = list(file_system.get_paths(recursive=True, upn=True)) # there are 24 subpaths in total self.assertEqual(len(paths), 24) @record def test_create_directory_from_file_system_client(self): # Arrange file_system = self._create_file_system() file_system.create_directory("dir1/dir2") paths = list(file_system.get_paths(recursive=False, upn=True)) self.assertEqual(len(paths), 1) self.assertEqual(paths[0].name, "dir1") @record def test_create_file_from_file_system_client(self): # Arrange file_system = self._create_file_system() file_system.create_file("dir1/dir2/file") paths = list(file_system.get_paths(recursive=True, upn=True)) self.assertEqual(len(paths), 3) self.assertEqual(paths[0].name, "dir1") self.assertEqual(paths[2].is_directory, False)
class FileSystemTest(StorageTestCase): def _setUp(self, account_name, account_key): url = self._get_account_url(account_name) self.dsc = DataLakeServiceClient(url, account_key) self.config = self.dsc._config self.test_file_systems = [] def tearDown(self): if not self.is_playback(): try: for file_system in self.test_file_systems: self.dsc.delete_file_system(file_system) except: pass return super(FileSystemTest, self).tearDown() # --Helpers----------------------------------------------------------------- def _get_file_system_reference(self, prefix=TEST_FILE_SYSTEM_PREFIX): file_system_name = self.get_resource_name(prefix) self.test_file_systems.append(file_system_name) return file_system_name def _create_file_system(self, file_system_prefix=TEST_FILE_SYSTEM_PREFIX): return self.dsc.create_file_system(self._get_file_system_reference(prefix=file_system_prefix)) # --Helpers----------------------------------------------------------------- @DataLakePreparer() def test_create_file_system(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange file_system_name = self._get_file_system_reference() # Act file_system_client = self.dsc.get_file_system_client(file_system_name) created = file_system_client.create_file_system() # Assert self.assertTrue(created) @DataLakePreparer() def test_file_system_exists(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange file_system_name = self._get_file_system_reference() # Act file_system_client1 = self.dsc.get_file_system_client(file_system_name) file_system_client2 = self.dsc.get_file_system_client("nonexistentfs") file_system_client1.create_file_system() self.assertTrue(file_system_client1.exists()) self.assertFalse(file_system_client2.exists()) @DataLakePreparer() def test_create_file_system_with_metadata(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange metadata = {'hello': 'world', 'number': '42'} file_system_name = self._get_file_system_reference() # Act file_system_client = self.dsc.get_file_system_client(file_system_name) created = file_system_client.create_file_system(metadata=metadata) # Assert meta = file_system_client.get_file_system_properties().metadata self.assertTrue(created) self.assertDictEqual(meta, metadata) @DataLakePreparer() def test_set_file_system_acl(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Act file_system = self._create_file_system() access_policy = AccessPolicy(permission=FileSystemSasPermissions(read=True), expiry=datetime.utcnow() + timedelta(hours=1), start=datetime.utcnow()) signed_identifier1 = {'testid': access_policy} response = file_system.set_file_system_access_policy(signed_identifier1, public_access=PublicAccess.FileSystem) self.assertIsNotNone(response.get('etag')) self.assertIsNotNone(response.get('last_modified')) acl1 = file_system.get_file_system_access_policy() self.assertIsNotNone(acl1['public_access']) self.assertEqual(len(acl1['signed_identifiers']), 1) # If set signed identifier without specifying the access policy then it will be default to None signed_identifier2 = {'testid': access_policy, 'test2': access_policy} file_system.set_file_system_access_policy(signed_identifier2) acl2 = file_system.get_file_system_access_policy() self.assertIsNone(acl2['public_access']) self.assertEqual(len(acl2['signed_identifiers']), 2) @DataLakePreparer() def test_list_file_systemss(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange file_system_name = self._get_file_system_reference() file_system = self.dsc.create_file_system(file_system_name) # Act file_systems = list(self.dsc.list_file_systems()) # Assert self.assertIsNotNone(file_systems) self.assertGreaterEqual(len(file_systems), 1) self.assertIsNotNone(file_systems[0]) self.assertNamedItemInContainer(file_systems, file_system.file_system_name) self.assertIsNotNone(file_systems[0].has_immutability_policy) self.assertIsNotNone(file_systems[0].has_legal_hold) @DataLakePreparer() def test_rename_file_system(self, datalake_storage_account_name, datalake_storage_account_key): if not self.is_playback(): return self._setUp(datalake_storage_account_name, datalake_storage_account_key) old_name1 = self._get_file_system_reference(prefix="oldcontainer1") old_name2 = self._get_file_system_reference(prefix="oldcontainer2") new_name = self._get_file_system_reference(prefix="newcontainer") filesystem1 = self.dsc.create_file_system(old_name1) self.dsc.create_file_system(old_name2) new_filesystem = self.dsc._rename_file_system(name=old_name1, new_name=new_name) with self.assertRaises(HttpResponseError): self.dsc._rename_file_system(name=old_name2, new_name=new_name) with self.assertRaises(HttpResponseError): filesystem1.get_file_system_properties() with self.assertRaises(HttpResponseError): self.dsc._rename_file_system(name="badfilesystem", new_name="filesystem") self.assertEqual(new_name, new_filesystem.get_file_system_properties().name) @DataLakePreparer() def test_rename_file_system_with_file_system_client(self, datalake_storage_account_name, datalake_storage_account_key): pytest.skip("Feature not yet enabled. Make sure to record this test once enabled.") self._setUp(datalake_storage_account_name, datalake_storage_account_key) old_name1 = self._get_file_system_reference(prefix="oldcontainer1") old_name2 = self._get_file_system_reference(prefix="oldcontainer2") new_name = self._get_file_system_reference(prefix="newcontainer") bad_name = self._get_file_system_reference(prefix="badcontainer") filesystem1 = self.dsc.create_file_system(old_name1) file_system2 = self.dsc.create_file_system(old_name2) bad_file_system = self.dsc.get_file_system_client(bad_name) new_filesystem = filesystem1._rename_file_system(new_name=new_name) with self.assertRaises(HttpResponseError): file_system2._rename_file_system(new_name=new_name) with self.assertRaises(HttpResponseError): filesystem1.get_file_system_properties() with self.assertRaises(HttpResponseError): bad_file_system._rename_file_system(new_name="filesystem") self.assertEqual(new_name, new_filesystem.get_file_system_properties().name) @DataLakePreparer() def test_rename_file_system_with_source_lease(self, datalake_storage_account_name, datalake_storage_account_key): if not self.is_playback(): return self._setUp(datalake_storage_account_name, datalake_storage_account_key) old_name = self._get_file_system_reference(prefix="old") new_name = self._get_file_system_reference(prefix="new") filesystem = self.dsc.create_file_system(old_name) filesystem_lease_id = filesystem.acquire_lease() with self.assertRaises(HttpResponseError): self.dsc._rename_file_system(name=old_name, new_name=new_name) with self.assertRaises(HttpResponseError): self.dsc._rename_file_system(name=old_name, new_name=new_name, lease="bad_id") new_filesystem = self.dsc._rename_file_system(name=old_name, new_name=new_name, lease=filesystem_lease_id) self.assertEqual(new_name, new_filesystem.get_file_system_properties().name) @DataLakePreparer() def test_undelete_file_system(self, datalake_storage_account_name, datalake_storage_account_key): # Needs soft delete enabled account. if not self.is_playback(): return self._setUp(datalake_storage_account_name, datalake_storage_account_key) name = self._get_file_system_reference() filesystem_client = self.dsc.create_file_system(name) # Act filesystem_client.delete_file_system() # to make sure the filesystem deleted with self.assertRaises(ResourceNotFoundError): filesystem_client.get_file_system_properties() filesystem_list = list(self.dsc.list_file_systems(include_deleted=True)) self.assertTrue(len(filesystem_list) >= 1) restored_version = 0 for filesystem in filesystem_list: # find the deleted filesystem and restore it if filesystem.deleted and filesystem.name == filesystem_client.file_system_name: restored_fs_client = self.dsc.undelete_file_system(filesystem.name, filesystem.deleted_version, new_name="restored" + name + str(restored_version)) restored_version += 1 # to make sure the deleted filesystem is restored props = restored_fs_client.get_file_system_properties() self.assertIsNotNone(props) @DataLakePreparer() def test_restore_to_existing_file_system(self, datalake_storage_account_name, datalake_storage_account_key): # Needs soft delete enabled account. if not self.is_playback(): return self._setUp(datalake_storage_account_name, datalake_storage_account_key) # get an existing filesystem existing_name = self._get_file_system_reference(prefix="existing2") name = self._get_file_system_reference(prefix="filesystem2") existing_filesystem_client = self.dsc.create_file_system(existing_name) filesystem_client = self.dsc.create_file_system(name) # Act filesystem_client.delete_file_system() # to make sure the filesystem deleted with self.assertRaises(ResourceNotFoundError): filesystem_client.get_file_system_properties() filesystem_list = list(self.dsc.list_file_systems(include_deleted=True)) self.assertTrue(len(filesystem_list) >= 1) for filesystem in filesystem_list: # find the deleted filesystem and restore it if filesystem.deleted and filesystem.name == filesystem_client.file_system_name: with self.assertRaises(HttpResponseError): self.dsc.undelete_file_system(filesystem.name, filesystem.deleted_version, new_name=existing_filesystem_client.file_system_name) @DataLakePreparer() def test_restore_file_system_with_sas(self, datalake_storage_account_name, datalake_storage_account_key): pytest.skip( "We are generating a SAS token therefore play only live but we also need a soft delete enabled account.") self._setUp(datalake_storage_account_name, datalake_storage_account_key) token = generate_account_sas( self.dsc.account_name, self.dsc.credential.account_key, ResourceTypes(service=True, file_system=True), AccountSasPermissions(read=True, write=True, list=True, delete=True), datetime.utcnow() + timedelta(hours=1), ) dsc = DataLakeServiceClient(self.dsc.url, token) name = self._get_file_system_reference(prefix="filesystem") filesystem_client = dsc.create_file_system(name) filesystem_client.delete_file_system() # to make sure the filesystem is deleted with self.assertRaises(ResourceNotFoundError): filesystem_client.get_file_system_properties() filesystem_list = list(dsc.list_file_systems(include_deleted=True)) self.assertTrue(len(filesystem_list) >= 1) restored_version = 0 for filesystem in filesystem_list: # find the deleted filesystem and restore it if filesystem.deleted and filesystem.name == filesystem_client.file_system_name: restored_fs_client = dsc.undelete_file_system(filesystem.name, filesystem.deleted_version, new_name="restored" + name + str(restored_version)) restored_version += 1 # to make sure the deleted filesystem is restored props = restored_fs_client.get_file_system_properties() self.assertIsNotNone(props) @DataLakePreparer() def test_delete_file_system_with_existing_file_system(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange file_system = self._create_file_system() # Act deleted = file_system.delete_file_system() # Assert self.assertIsNone(deleted) @DataLakePreparer() def test_delete_none_existing_file_system(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) fake_file_system_client = self.dsc.get_file_system_client("fakeclient") # Act with self.assertRaises(ResourceNotFoundError): fake_file_system_client.delete_file_system(match_condition=MatchConditions.IfMissing) @DataLakePreparer() def test_list_file_systems_with_include_metadata(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange file_system = self._create_file_system() metadata = {'hello': 'world', 'number': '42'} resp = file_system.set_file_system_metadata(metadata) # Act file_systems = list(self.dsc.list_file_systems( name_starts_with=file_system.file_system_name, include_metadata=True)) # Assert self.assertIsNotNone(file_systems) self.assertGreaterEqual(len(file_systems), 1) self.assertIsNotNone(file_systems[0]) self.assertNamedItemInContainer(file_systems, file_system.file_system_name) self.assertDictEqual(file_systems[0].metadata, metadata) @DataLakePreparer() def test_list_file_systems_by_page(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange for i in range(0, 6): self._create_file_system(file_system_prefix="filesystem{}".format(i)) # Act file_systems = list(next(self.dsc.list_file_systems( results_per_page=3, name_starts_with="file", include_metadata=True).by_page())) # Assert self.assertIsNotNone(file_systems) self.assertGreaterEqual(len(file_systems), 3) @DataLakePreparer() def test_list_file_systems_with_public_access(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange file_system_name = self._get_file_system_reference() file_system = self.dsc.get_file_system_client(file_system_name) file_system.create_file_system(public_access="blob") metadata = {'hello': 'world', 'number': '42'} resp = file_system.set_file_system_metadata(metadata) # Act file_systems = list(self.dsc.list_file_systems( name_starts_with=file_system.file_system_name, include_metadata=True)) # Assert self.assertIsNotNone(file_systems) self.assertGreaterEqual(len(file_systems), 1) self.assertIsNotNone(file_systems[0]) self.assertNamedItemInContainer(file_systems, file_system.file_system_name) self.assertDictEqual(file_systems[0].metadata, metadata) self.assertTrue(file_systems[0].public_access is PublicAccess.File) @DataLakePreparer() def test_get_file_system_properties(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange metadata = {'hello': 'world', 'number': '42'} file_system = self._create_file_system() file_system.set_file_system_metadata(metadata) # Act props = file_system.get_file_system_properties() # Assert self.assertIsNotNone(props) self.assertDictEqual(props.metadata, metadata) self.assertIsNotNone(props.has_immutability_policy) self.assertIsNotNone(props.has_legal_hold) @DataLakePreparer() def test_service_client_session_closes_after_filesystem_creation(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange dsc2 = DataLakeServiceClient(self.dsc.url, credential=datalake_storage_account_key) with DataLakeServiceClient(self.dsc.url, credential=datalake_storage_account_key) as ds_client: fs1 = ds_client.create_file_system(self._get_file_system_reference(prefix="fs1")) fs1.delete_file_system() dsc2.create_file_system(self._get_file_system_reference(prefix="fs2")) dsc2.close() @DataLakePreparer() def test_list_paths(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange file_system = self._create_file_system() for i in range(0, 6): file_system.create_directory("dir1{}".format(i)) paths = list(file_system.get_paths(upn=True)) self.assertEqual(len(paths), 6) self.assertTrue(isinstance(paths[0].last_modified, datetime)) @DataLakePreparer() def test_list_paths_which_are_all_files(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange file_system = self._create_file_system() for i in range(0, 6): file_system.create_file("file{}".format(i)) paths = list(file_system.get_paths(upn=True)) self.assertEqual(len(paths), 6) @DataLakePreparer() def test_list_paths_with_max_per_page(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange file_system = self._create_file_system() for i in range(0, 6): file_system.create_directory("dir1{}".format(i)) generator1 = file_system.get_paths(max_results=2, upn=True).by_page() paths1 = list(next(generator1)) generator2 = file_system.get_paths(max_results=4, upn=True)\ .by_page(continuation_token=generator1.continuation_token) paths2 = list(next(generator2)) self.assertEqual(len(paths1), 2) self.assertEqual(len(paths2), 4) @DataLakePreparer() def test_list_paths_under_specific_path(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange file_system = self._create_file_system() for i in range(0, 6): file_system.create_directory("dir1{}".format(i)) # create a subdirectory under the current directory subdir = file_system.get_directory_client("dir1{}".format(i)).create_sub_directory("subdir") subdir.create_sub_directory("subsub") # create a file under the current directory file_client = subdir.create_file("file") file_client.append_data(b"abced", 0, 5) file_client.flush_data(5) generator1 = file_system.get_paths(path="dir10/subdir", max_results=2, upn=True).by_page() paths = list(next(generator1)) self.assertEqual(len(paths), 2) self.assertEqual(paths[0].content_length, 5) @DataLakePreparer() def test_list_paths_recursively(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange file_system = self._create_file_system() for i in range(0, 6): file_system.create_directory("dir1{}".format(i)) # create a subdirectory under the current directory subdir = file_system.get_directory_client("dir1{}".format(i)).create_sub_directory("subdir") subdir.create_sub_directory("subsub") # create a file under the current directory subdir.create_file("file") paths = list(file_system.get_paths(recursive=True, upn=True)) # there are 24 subpaths in total self.assertEqual(len(paths), 24) @DataLakePreparer() def test_list_paths_pages_correctly(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange file_system = self._create_file_system(file_system_prefix="fs1") for i in range(0, 6): file_system.create_directory("dir1{}".format(i)) for i in range(0, 6): file_system.create_file("file{}".format(i)) generator = file_system.get_paths(max_results=6, upn=True).by_page() paths1 = list(next(generator)) paths2 = list(next(generator)) with self.assertRaises(StopIteration): list(next(generator)) self.assertEqual(len(paths1), 6) self.assertEqual(len(paths2), 6) @DataLakePreparer() def test_create_directory_from_file_system_client(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange file_system = self._create_file_system() file_system.create_directory("dir1/dir2") paths = list(file_system.get_paths(recursive=False, upn=True)) self.assertEqual(len(paths), 1) self.assertEqual(paths[0].name, "dir1") @DataLakePreparer() def test_create_file_from_file_system_client(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange file_system = self._create_file_system() file_system.create_file("dir1/dir2/file") paths = list(file_system.get_paths(recursive=True, upn=True)) self.assertEqual(len(paths), 3) self.assertEqual(paths[0].name, "dir1") self.assertEqual(paths[2].is_directory, False) @DataLakePreparer() def test_get_root_directory_client(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) file_system = self._create_file_system() directory_client = file_system._get_root_directory_client() acl = 'user::rwx,group::r-x,other::rwx' directory_client.set_access_control(acl=acl) access_control = directory_client.get_access_control() self.assertEqual(acl, access_control['acl']) @DataLakePreparer() def test_file_system_sessions_closes_properly(self, datalake_storage_account_name, datalake_storage_account_key): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange file_system_client = self._create_file_system("fenrhxsbfvsdvdsvdsadb") with file_system_client as fs_client: with fs_client.get_file_client("file1.txt") as f_client: f_client.create_file() with fs_client.get_file_client("file2.txt") as f_client: f_client.create_file() with fs_client.get_directory_client("file1") as f_client: f_client.create_directory() with fs_client.get_directory_client("file2") as f_client: f_client.create_directory()
class DirectoryTest(StorageTestCase): def setUp(self): super(DirectoryTest, self).setUp() url = self._get_account_url() self.dsc = DataLakeServiceClient( url, credential=self.settings.STORAGE_DATA_LAKE_ACCOUNT_KEY) self.config = self.dsc._config self.file_system_name = self.get_resource_name('filesystem') if not self.is_playback(): file_system = self.dsc.get_file_system_client( self.file_system_name) try: file_system.create_file_system(timeout=5) except ResourceExistsError: pass def tearDown(self): if not self.is_playback(): try: self.dsc.delete_file_system(self.file_system_name) for file_system in self.dsc.list_file_systems(): self.dsc.delete_file_system(file_system.name) except: pass return super(DirectoryTest, self).tearDown() # --Helpers----------------------------------------------------------------- def _get_directory_reference(self, prefix=TEST_DIRECTORY_PREFIX): directory_name = self.get_resource_name(prefix) return directory_name def _create_directory_and_get_directory_client(self, directory_name=None): directory_name = directory_name if directory_name else self._get_directory_reference( ) directory_client = self.dsc.get_directory_client( self.file_system_name, directory_name) directory_client.create_directory() return directory_client def _create_file_system(self): return self.dsc.create_file_system(self._get_file_system_reference()) # --Helpers----------------------------------------------------------------- @record def test_create_directory(self): # Arrange directory_name = self._get_directory_reference() content_settings = ContentSettings(content_language='spanish', content_disposition='inline') # Act directory_client = self.dsc.get_directory_client( self.file_system_name, directory_name) created = directory_client.create_directory( content_settings=content_settings) # Assert self.assertTrue(created) @record def test_using_oauth_token_credential_to_create_directory(self): # generate a token with directory level create permission directory_name = self._get_directory_reference() token_credential = self.generate_oauth_token() directory_client = DataLakeDirectoryClient(self.dsc.url, self.file_system_name, directory_name, credential=token_credential) response = directory_client.create_directory() self.assertIsNotNone(response) @record def test_create_directory_with_match_conditions(self): # Arrange directory_name = self._get_directory_reference() # Act directory_client = self.dsc.get_directory_client( self.file_system_name, directory_name) created = directory_client.create_directory( match_condition=MatchConditions.IfMissing) # Assert self.assertTrue(created) @record def test_create_directory_with_permission(self): # Arrange directory_name = self._get_directory_reference() # Act directory_client = self.dsc.get_directory_client( self.file_system_name, directory_name) created = directory_client.create_directory(permissions="rwxr--r--", umask="0000") prop = directory_client.get_access_control() # Assert self.assertTrue(created) self.assertEqual(prop['permissions'], 'rwxr--r--') @record def test_create_directory_with_content_settings(self): # Arrange directory_name = self._get_directory_reference() content_settings = ContentSettings(content_language='spanish', content_disposition='inline') # Act directory_client = self.dsc.get_directory_client( self.file_system_name, directory_name) created = directory_client.create_directory( content_settings=content_settings) # Assert self.assertTrue(created) @record def test_create_directory_with_metadata(self): # Arrange directory_name = self._get_directory_reference() metadata = {'hello': 'world', 'number': '42'} # Act directory_client = self.dsc.get_directory_client( self.file_system_name, directory_name) created = directory_client.create_directory(metadata=metadata) properties = directory_client.get_directory_properties() # Assert self.assertTrue(created) @record def test_delete_directory(self): # Arrange directory_name = self._get_directory_reference() metadata = {'hello': 'world', 'number': '42'} directory_client = self.dsc.get_directory_client( self.file_system_name, directory_name) directory_client.create_directory(metadata=metadata) response = directory_client.delete_directory() # Assert self.assertIsNone(response) @record def test_delete_directory_with_if_modified_since(self): # Arrange directory_name = self._get_directory_reference() directory_client = self.dsc.get_directory_client( self.file_system_name, directory_name) directory_client.create_directory() prop = directory_client.get_directory_properties() with self.assertRaises(ResourceModifiedError): directory_client.delete_directory( if_modified_since=prop['last_modified']) @record def test_create_sub_directory_and_delete_sub_directory(self): # Arrange directory_name = self._get_directory_reference() metadata = {'hello': 'world', 'number': '42'} # Create a directory first, to prepare for creating sub directory directory_client = self.dsc.get_directory_client( self.file_system_name, directory_name) directory_client.create_directory(metadata=metadata) # Create sub directory from the current directory sub_directory_name = 'subdir' sub_directory_created = directory_client.create_sub_directory( sub_directory_name) # to make sure the sub directory was indeed created by get sub_directory properties from sub directory client sub_directory_client = self.dsc.get_directory_client( self.file_system_name, directory_name + '/' + sub_directory_name) sub_properties = sub_directory_client.get_directory_properties() # Assert self.assertTrue(sub_directory_created) self.assertTrue(sub_properties) # Act directory_client.delete_sub_directory(sub_directory_name) with self.assertRaises(ResourceNotFoundError): sub_directory_client.get_directory_properties() @record def test_set_access_control(self): directory_name = self._get_directory_reference() metadata = {'hello': 'world', 'number': '42'} directory_client = self.dsc.get_directory_client( self.file_system_name, directory_name) directory_client.create_directory(metadata=metadata) response = directory_client.set_access_control(permissions='0777') # Assert self.assertIsNotNone(response) @record def test_set_access_control_with_acl(self): directory_name = self._get_directory_reference() metadata = {'hello': 'world', 'number': '42'} directory_client = self.dsc.get_directory_client( self.file_system_name, directory_name) directory_client.create_directory(metadata=metadata) acl = 'user::rwx,group::r-x,other::rwx' directory_client.set_access_control(acl=acl) access_control = directory_client.get_access_control() # Assert self.assertIsNotNone(access_control) self.assertEqual(acl, access_control['acl']) @record def test_set_access_control_if_none_modified(self): directory_name = self._get_directory_reference() directory_client = self.dsc.get_directory_client( self.file_system_name, directory_name) resp = directory_client.create_directory() response = directory_client.set_access_control( permissions='0777', etag=resp['etag'], match_condition=MatchConditions.IfNotModified) # Assert self.assertIsNotNone(response) @record def test_get_access_control(self): directory_name = self._get_directory_reference() metadata = {'hello': 'world', 'number': '42'} directory_client = self.dsc.get_directory_client( self.file_system_name, directory_name) directory_client.create_directory(metadata=metadata, permissions='0777') # Act response = directory_client.get_access_control() # Assert self.assertIsNotNone(response) @record def test_get_access_control_with_match_conditions(self): directory_name = self._get_directory_reference() directory_client = self.dsc.get_directory_client( self.file_system_name, directory_name) resp = directory_client.create_directory(permissions='0777', umask='0000') # Act response = directory_client.get_access_control( etag=resp['etag'], match_condition=MatchConditions.IfNotModified) # Assert self.assertIsNotNone(response) self.assertEquals(response['permissions'], 'rwxrwxrwx') @record def test_rename_from(self): metadata = {'hello': 'world', 'number': '42'} directory_name = self._get_directory_reference() directory_client = self.dsc.get_directory_client( self.file_system_name, directory_name) directory_client.create_directory() new_name = "newname" new_directory_client = self.dsc.get_directory_client( self.file_system_name, new_name) new_directory_client._rename_path('/' + self.file_system_name + '/' + directory_name, metadata=metadata) properties = new_directory_client.get_directory_properties() self.assertIsNotNone(properties) @record def test_rename_from_a_shorter_directory_to_longer_directory(self): # TODO: investigate why rename shorter path to a longer one does not work pytest.skip("") directory_name = self._get_directory_reference() self._create_directory_and_get_directory_client(directory_name="old") new_name = "newname" new_directory_client = self._create_directory_and_get_directory_client( directory_name=new_name) new_directory_client = new_directory_client.create_sub_directory( "newsub") new_directory_client._rename_path('/' + self.file_system_name + '/' + directory_name) properties = new_directory_client.get_directory_properties() self.assertIsNotNone(properties) @record def test_rename_from_a_directory_in_another_file_system(self): # create a file dir1 under file system1 old_file_system_name = "oldfilesystem" old_dir_name = "olddir" old_client = self.dsc.get_file_system_client(old_file_system_name) old_client.create_file_system() old_client.create_directory(old_dir_name) # create a dir2 under file system2 new_name = "newname" new_directory_client = self._create_directory_and_get_directory_client( directory_name=new_name) new_directory_client = new_directory_client.create_sub_directory( "newsub") # rename dir1 under file system1 to dir2 under file system2 new_directory_client._rename_path('/' + old_file_system_name + '/' + old_dir_name) properties = new_directory_client.get_directory_properties() self.assertIsNotNone(properties) self.dsc.delete_file_system(old_file_system_name) @record def test_rename_to_an_existing_directory_in_another_file_system(self): # create a file dir1 under file system1 destination_file_system_name = "destfilesystem" destination_dir_name = "destdir" fs_client = self.dsc.get_file_system_client( destination_file_system_name) fs_client.create_file_system() destination_directory_client = fs_client.create_directory( destination_dir_name) # create a dir2 under file system2 source_name = "source" source_directory_client = self._create_directory_and_get_directory_client( directory_name=source_name) source_directory_client = source_directory_client.create_sub_directory( "subdir") # rename dir2 under file system2 to dir1 under file system1 res = source_directory_client.rename_directory( '/' + destination_file_system_name + '/' + destination_dir_name) # the source directory has been renamed to destination directory, so it cannot be found with self.assertRaises(HttpResponseError): source_directory_client.get_directory_properties() self.assertEquals(res.url, destination_directory_client.url) @record def test_rename_with_none_existing_destination_condition_and_source_unmodified_condition( self): non_existing_dir_name = "nonexistingdir" # create a file system1 destination_file_system_name = self._get_directory_reference( "destfilesystem") fs_client = self.dsc.get_file_system_client( destination_file_system_name) fs_client.create_file_system() # create a dir2 under file system2 source_name = "source" source_directory_client = self._create_directory_and_get_directory_client( directory_name=source_name) source_directory_client = source_directory_client.create_sub_directory( "subdir") # rename dir2 under file system2 to a non existing directory under file system1, # when dir1 does not exist and dir2 wasn't modified etag = source_directory_client.get_directory_properties()['etag'] res = source_directory_client.rename_directory( '/' + destination_file_system_name + '/' + non_existing_dir_name, match_condition=MatchConditions.IfMissing, source_etag=etag, source_match_condition=MatchConditions.IfNotModified) # the source directory has been renamed to destination directory, so it cannot be found with self.assertRaises(HttpResponseError): source_directory_client.get_directory_properties() self.assertEquals(non_existing_dir_name, res.path_name) @record def test_rename_to_an_non_existing_directory_in_another_file_system(self): # create a file dir1 under file system1 destination_file_system_name = self._get_directory_reference( "destfilesystem") non_existing_dir_name = "nonexistingdir" fs_client = self.dsc.get_file_system_client( destination_file_system_name) fs_client.create_file_system() # create a dir2 under file system2 source_name = "source" source_directory_client = self._create_directory_and_get_directory_client( directory_name=source_name) source_directory_client = source_directory_client.create_sub_directory( "subdir") # rename dir2 under file system2 to dir1 under file system1 res = source_directory_client.rename_directory( '/' + destination_file_system_name + '/' + non_existing_dir_name) # the source directory has been renamed to destination directory, so it cannot be found with self.assertRaises(HttpResponseError): source_directory_client.get_directory_properties() self.assertEquals(non_existing_dir_name, res.path_name) @record def test_rename_directory_to_non_empty_directory(self): # TODO: investigate why rename non empty dir doesn't work pytest.skip("") dir1 = self._create_directory_and_get_directory_client("dir1") dir1.create_sub_directory("subdir") dir2 = self._create_directory_and_get_directory_client("dir2") dir2.rename_directory(dir1.file_system_name + '/' + dir1.path_name) with self.assertRaises(HttpResponseError): dir2.get_directory_properties() @record def test_get_properties(self): # Arrange directory_name = self._get_directory_reference() metadata = {'hello': 'world', 'number': '42'} directory_client = self.dsc.get_directory_client( self.file_system_name, directory_name) directory_client.create_directory(metadata=metadata) properties = directory_client.get_directory_properties() # Assert self.assertTrue(properties) self.assertIsNotNone(properties.metadata) self.assertEqual(properties.metadata['hello'], metadata['hello']) @record def test_using_directory_sas_to_read(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return client = self._create_directory_and_get_directory_client() directory_name = client.path_name # generate a token with directory level read permission token = generate_directory_sas( self.dsc.account_name, self.file_system_name, directory_name, account_key=self.dsc.credential.account_key, permission=DirectorySasPermissions(read=True), expiry=datetime.utcnow() + timedelta(hours=1), ) directory_client = DataLakeDirectoryClient(self.dsc.url, self.file_system_name, directory_name, credential=token) access_control = directory_client.get_access_control() self.assertIsNotNone(access_control) @record def test_using_directory_sas_to_create(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # generate a token with directory level create permission directory_name = self._get_directory_reference() token = generate_directory_sas( self.dsc.account_name, self.file_system_name, directory_name, account_key=self.dsc.credential.account_key, permission=DirectorySasPermissions(create=True), expiry=datetime.utcnow() + timedelta(hours=1), ) directory_client = DataLakeDirectoryClient(self.dsc.url, self.file_system_name, directory_name, credential=token) response = directory_client.create_directory() self.assertIsNotNone(response)
def data_lake_service_sample(self): # Instantiate a DataLakeServiceClient using a connection string # [START create_datalake_service_client] from azure.storage.filedatalake import DataLakeServiceClient datalake_service_client = DataLakeServiceClient.from_connection_string( self.connection_string) # [END create_datalake_service_client] # Instantiate a DataLakeServiceClient Azure Identity credentials. # [START create_datalake_service_client_oauth] from azure.identity import ClientSecretCredential token_credential = ClientSecretCredential( self.active_directory_tenant_id, self.active_directory_application_id, self.active_directory_application_secret, ) datalake_service_client = DataLakeServiceClient( "https://{}.dfs.core.windows.net".format(self.account_name), credential=token_credential) # [END create_datalake_service_client_oauth] # get user delegation key # [START get_user_delegation_key] from datetime import datetime, timedelta user_delegation_key = datalake_service_client.get_user_delegation_key( datetime.utcnow(), datetime.utcnow() + timedelta(hours=1)) # [END get_user_delegation_key] # Create file systems # [START create_file_system_from_service_client] datalake_service_client.create_file_system("filesystem") # [END create_file_system_from_service_client] file_system_client = datalake_service_client.create_file_system( "anotherfilesystem") # List file systems # [START list_file_systems] file_systems = datalake_service_client.list_file_systems() for file_system in file_systems: print(file_system.name) # [END list_file_systems] # Get Clients from DataLakeServiceClient file_system_client = datalake_service_client.get_file_system_client( file_system_client.file_system_name) # [START get_directory_client_from_service_client] directory_client = datalake_service_client.get_directory_client( file_system_client.file_system_name, "mydirectory") # [END get_directory_client_from_service_client] # [START get_file_client_from_service_client] file_client = datalake_service_client.get_file_client( file_system_client.file_system_name, "myfile") # [END get_file_client_from_service_client] # Create file and set properties metadata = {'hello': 'world', 'number': '42'} from azure.storage.filedatalake import ContentSettings content_settings = ContentSettings(content_language='spanish', content_disposition='inline') file_client.create_file(content_settings=content_settings) file_client.set_metadata(metadata=metadata) file_props = file_client.get_file_properties() print(file_props.metadata) # Create file/directory and set properties directory_client.create_directory(content_settings=content_settings, metadata=metadata) dir_props = directory_client.get_directory_properties() print(dir_props.metadata) # Delete File Systems # [START delete_file_system_from_service_client] datalake_service_client.delete_file_system("filesystem") # [END delete_file_system_from_service_client] file_system_client.delete_file_system()
# In no event shall Microsoft, its authors, owners of this repository or anyone else involved in the creation, production, or delivery of the scripts be liable for any damages whatsoever (including, # without limitation, damages for loss of business profits, business interruption, loss of business information, or other pecuniary loss) arising out of the use of or inability to use the sample scripts or documentation, even if Microsoft has been advised of the possibility of such damages #------------------------------------------------------------------------- #IMPORT THE LIBRARIES INTO YOUR FILE from azure.storage.filedatalake import DataLakeServiceClient from azure.storage.filedatalake._shared.base_client import create_configuration #OPTION 1 - MAKING USE OF CONNECTION STRING AND CREATING THE DATALAKE CLIENT connection_string = "PUT CONNECTION STRING HERE" #CREATE THE DATALAKE SERVICE CLIENT service_client = DataLakeServiceClient.from_connection_string( connection_string) ##OPTION 2 - MAKING USE OF ACCESS KEY AND CREATING THE DATALAKE CLIENT storage_account_key = "ACCESS KEY" storage_account_name = "ACCOUNT NAME" #CREATE THE DATALAKE SERVICE CLIENT service_client = DataLakeServiceClient( account_url="{}://{}.dfs.core.windows.net".format("https", storage_account_name), credential=storage_account_key) #PERFORM THE LISTING OPERATION file_systems = service_client.list_file_systems() for file_system in file_systems: print(file_system.name)
class Adl(object): def __init__(self): url = self.datalake_account_url() key = self.datalake_account_key() print('url: {}'.format(url)) print('key: {}'.format(key)) self.service_client = DataLakeServiceClient(account_url=url, credential=key) print(self.service_client) def create_fs(self, fsname): try: fs_client = self.service_client.get_file_system_client(fsname) fs_client.create_file_system() print('create_fs: {}'.format(fsname)) except ResourceExistsError: pass def delete_fs(self, fsname): try: fs_client = self.service_client.get_file_system_client(fsname) fs_client.delete_file_system() except ResourceNotFoundError: pass def filesystem_list(self): return self.service_client.list_file_systems() def create_dir(self, fsname, dirname): try: fs_client = self.service_client.get_file_system_client(fsname) fs_client.create_directory(dirname) print('create_dir: {} in fs: {}'.format(dirname, fsname)) except: pass def file_list(self, fsname, dirname): try: fsc = self.service_client.get_file_system_client( file_system=fsname) return fsc.get_paths(path=dirname) except Exception as e: return list() def directory_client(self, fsname, dirname): try: fs_client = self.service_client.get_file_system_client(fsname) return fs_client.get_directory_client(dirname) except: return None def upload_file(self, dir_client, local_path, remote_name, opts={}): file_client = dir_client.create_file(remote_name) local_file = open(local_path, 'r') file_contents = local_file.read() print('upload_file, opts: {}'.format(opts)) # https://docs.microsoft.com/en-us/python/api/azure-storage-file-datalake/azure.storage.filedatalake.contentsettings?view=azure-python cs = ContentSettings(**opts) file_client.upload_data(file_contents, overwrite=True, content_settings=cs) print('upload_file; {} -> {}'.format(local_path, remote_name)) def download_file(self, dir_client, remote_name, local_path): file_client = dir_client.get_file_client(remote_name) download = file_client.download_file() downloaded_bytes = download.readall() local_file = open(local_path, 'wb') local_file.write(downloaded_bytes) local_file.close() print('download_file; {} -> {}'.format(remote_name, local_path)) def datalake_account_url(self): storage_account_name = os.environ['AZURE_ADL_ACCOUNT'] return "{}://{}.dfs.core.windows.net".format("https", storage_account_name) def datalake_account_key(self): return os.environ['AZURE_ADL_KEY'] def datalake_account_conn_string(self): return os.environ['AZURE_ADL_CONNECTION_STRING']
class FileSystemTest(StorageTestCase): def setUp(self): super(FileSystemTest, self).setUp() url = self._get_account_url() self.dsc = DataLakeServiceClient(url, credential=self.settings.STORAGE_DATA_LAKE_ACCOUNT_KEY) self.config = self.dsc._config self.test_file_systems = [] def tearDown(self): if not self.is_playback(): try: for file_system in self.test_file_systems: self.dsc.delete_file_system(file_system) except: pass return super(FileSystemTest, self).tearDown() # --Helpers----------------------------------------------------------------- def _get_file_system_reference(self, prefix=TEST_FILE_SYSTEM_PREFIX): file_system_name = self.get_resource_name(prefix) self.test_file_systems.append(file_system_name) return file_system_name def _create_file_system(self, file_system_prefix=TEST_FILE_SYSTEM_PREFIX): return self.dsc.create_file_system(self._get_file_system_reference(prefix=file_system_prefix)) # --Helpers----------------------------------------------------------------- @record def test_create_file_system(self): # Arrange file_system_name = self._get_file_system_reference() # Act file_system_client = self.dsc.get_file_system_client(file_system_name) created = file_system_client.create_file_system() # Assert self.assertTrue(created) @record def test_list_file_systemss(self): # Arrange file_system_name = self._get_file_system_reference() file_system = self.dsc.create_file_system(file_system_name) # Act file_systems = list(self.dsc.list_file_systems()) # Assert self.assertIsNotNone(file_systems) self.assertGreaterEqual(len(file_systems), 1) self.assertIsNotNone(file_systems[0]) self.assertNamedItemInContainer(file_systems, file_system.file_system_name) self.assertIsNotNone(file_systems[0].has_immutability_policy) self.assertIsNotNone(file_systems[0].has_legal_hold) @record def test_delete_file_system_with_existing_file_system(self): # Arrange file_system = self._create_file_system() # Act deleted = file_system.delete_file_system() # Assert self.assertIsNone(deleted) @record def test_list_file_systems_with_include_metadata(self): # Arrange file_system = self._create_file_system() metadata = {'hello': 'world', 'number': '42'} resp = file_system.set_file_system_metadata(metadata) # Act file_systems = list(self.dsc.list_file_systems( name_starts_with=file_system.file_system_name, include_metadata=True)) # Assert self.assertIsNotNone(file_systems) self.assertGreaterEqual(len(file_systems), 1) self.assertIsNotNone(file_systems[0]) self.assertNamedItemInContainer(file_systems, file_system.file_system_name) self.assertDictEqual(file_systems[0].metadata, metadata) @record def test_list_file_systems_by_page(self): # Arrange for i in range(0, 6): self._create_file_system(file_system_prefix="filesystem{}".format(i)) # Act file_systems = list(next(self.dsc.list_file_systems( results_per_page=3, name_starts_with="file", include_metadata=True).by_page())) # Assert self.assertIsNotNone(file_systems) self.assertGreaterEqual(len(file_systems), 3) @record def test_list_file_systems_with_public_access(self): # Arrange file_system_name = self._get_file_system_reference() file_system = self.dsc.get_file_system_client(file_system_name) file_system.create_file_system(public_access="blob") metadata = {'hello': 'world', 'number': '42'} resp = file_system.set_file_system_metadata(metadata) # Act file_systems = list(self.dsc.list_file_systems( name_starts_with=file_system.file_system_name, include_metadata=True)) # Assert self.assertIsNotNone(file_systems) self.assertGreaterEqual(len(file_systems), 1) self.assertIsNotNone(file_systems[0]) self.assertNamedItemInContainer(file_systems, file_system.file_system_name) self.assertDictEqual(file_systems[0].metadata, metadata) self.assertTrue(file_systems[0].public_access is PublicAccess.File) @record def test_get_file_system_properties(self): # Arrange metadata = {'hello': 'world', 'number': '42'} file_system = self._create_file_system() file_system.set_file_system_metadata(metadata) # Act props = file_system.get_file_system_properties() # Assert self.assertIsNotNone(props) self.assertDictEqual(props.metadata, metadata) self.assertIsNotNone(props.has_immutability_policy) self.assertIsNotNone(props.has_legal_hold) @record def test_list_paths(self): # Arrange file_system = self._create_file_system() for i in range(0, 6): file_system.create_directory("dir1{}".format(i)) paths = list(file_system.get_paths(upn=True)) self.assertEqual(len(paths), 6) @record def test_list_paths_with_max_per_page(self): # Arrange file_system = self._create_file_system() for i in range(0, 6): file_system.create_directory("dir1{}".format(i)) generator1 = file_system.get_paths(max_results=2, upn=True).by_page() paths1 = list(next(generator1)) generator2 = file_system.get_paths(max_results=4, upn=True)\ .by_page(continuation_token=generator1.continuation_token) paths2 = list(next(generator2)) self.assertEqual(len(paths1), 2) self.assertEqual(len(paths2), 4) @record def test_list_paths_under_specific_path(self): # Arrange file_system = self._create_file_system() for i in range(0, 6): file_system.create_directory("dir1{}".format(i)) # create a subdirectory under the current directory subdir = file_system.get_directory_client("dir1{}".format(i)).create_sub_directory("subdir") subdir.create_sub_directory("subsub") # create a file under the current directory file_client = subdir.create_file("file") file_client.append_data(b"abced", 0, 5) file_client.flush_data(5) generator1 = file_system.get_paths(path="dir10/subdir", max_results=2, upn=True).by_page() paths = list(next(generator1)) self.assertEqual(len(paths), 2) self.assertEqual(paths[0].content_length, 5)