def test_path_decoder_error(self):
path = "https://aaa/bbb/ccc"
bpd = BlobPathDecoder()
with pytest.raises(AzfsInvalidPathError):
bpd.decode(path)
with pytest.raises(AzfsInvalidPathError):
BlobPathDecoder(path)
def test_add_pattern_error(self):
# shortage of %
path_pattern = "%A=%T=%C"
with pytest.raises(AzfsInputError):
BlobPathDecoder.add_pattern(pattern=path_pattern)
# unknown %
path_pattern = "%A=%T=%C%Z"
with pytest.raises(AzfsInputError):
BlobPathDecoder.add_pattern(pattern=path_pattern)
def test_add_pattern(self, path_pattern, path, storage_account_name,
account_type, container_name, blob_file):
# add new pattern
BlobPathDecoder.add_pattern(pattern=path_pattern)
storage_account_name_v, account_type_v, container_name_v, blob_file_v = BlobPathDecoder(
path).get()
assert storage_account_name_v == storage_account_name
assert account_type_v == account_type
assert container_name_v == container_name
assert blob_file_v == blob_file
def test_path_decoder_pass(self, path, storage_account_name, account_type,
container_name, blob_file):
bpd = BlobPathDecoder()
storage_account_name_v, account_type_v, container_name_v, blob_file_v = bpd.decode(
path).get()
assert storage_account_name_v == storage_account_name
assert account_type_v == account_type
assert container_name_v == container_name
assert blob_file_v == blob_file
storage_account_name_v, account_type_v, container_name_v, blob_file_v = BlobPathDecoder(
path).get()
assert storage_account_name_v == storage_account_name
assert account_type_v == account_type
assert container_name_v == container_name
assert blob_file_v == blob_file
def rm(self, path: str) -> bool:
"""
delete the file in blob
:param path:
:return:
"""
_, account_kind, _, _ = BlobPathDecoder(path).get_with_url()
return AzfsClient.get(account_kind, credential=self.credential).rm(path=path)
def _upload_data(self, path: str, data) -> bool:
"""
upload data to blob or data_lake storage account
:param path:
:param data:
:return:
"""
_, account_kind, _, _ = BlobPathDecoder(path).get_with_url()
return AzfsClient.get(account_kind, credential=self.credential).upload_data(path=path, data=data)
def _download_data(self, path: str) -> Union[bytes, str, io.BytesIO]:
"""
storage accountのタイプによってfile_clientを変更し、データを取得する関数
特定のファイルを取得する関数
:param path:
:return:
"""
_, account_kind, _, _ = BlobPathDecoder(path).get_with_url()
return AzfsClient.get(account_kind, credential=self.credential).download_data(path=path)
def ls(self, path: str):
"""
list blob file
:param path:
:return:
"""
_, account_kind, _, file_path = BlobPathDecoder(path).get_with_url()
file_list = AzfsClient.get(account_kind, credential=self.credential).ls(path=path)
return ls_filter(file_path_list=file_list, file_path=file_path)
def get_properties(self, path: str) -> dict:
"""
get file properties, such as
* name
* creation_time
* last_modified_time
* size
* content_hash(md5)
:param path:
:return:
"""
_, account_kind, _, _ = BlobPathDecoder(path).get_with_url()
return AzfsClient.get(account_kind, credential=self.credential).get_properties(path=path)
def get_container_client_from_path(
self, path) -> Union[ContainerClient, FileSystemClient]:
"""
get container_client from given path
:param path:
:return:
"""
storage_account_url, _, file_system, _ = BlobPathDecoder(
path).get_with_url()
return self._get_container_client(
storage_account_url=storage_account_url,
file_system=file_system,
credential=self.credential)
def get_file_client_from_path(
self, path) -> Union[BlobClient, DataLakeFileClient]:
"""
get file_client from given path
:param path:
:return:
"""
storage_account_url, account_kind, file_system, file_path = BlobPathDecoder(
path).get_with_url()
return self._get_file_client(storage_account_url=storage_account_url,
file_system=file_system,
file_path=file_path,
credential=self.credential)
def get_container_client_from_path(self, path: str) -> FileSystemClientType:
"""
get container_client from given path
Args:
path: Azure path that ``BlobPathDecode()`` can decode
Returns:
Union[ContainerClient, FileSystemClient]
"""
account_url, _, file_system, _ = BlobPathDecoder(path).get_with_url()
return self._get_container_client(
account_url=account_url,
file_system=file_system)
def get_file_client_from_path(self, path: str) -> FileClientType:
"""
get file_client from given path
Args:
path: Azure path that ``BlobPathDecode()`` can decode
Returns:
Union[BlobClient, DataLakeFileClient, QueueClient]
"""
account_url, account_kind, file_system, file_path = BlobPathDecoder(path).get_with_url()
return self._get_file_client(
account_url=account_url,
file_system=file_system,
file_path=file_path)