def __init__(self, bucket_prefix: str, cache_path: str = None) -> None:
'''
Abstraction of storage of files on Azure storage accounts. Do not call
this constructor but call the AzureFileStorage.setup() factory method
:param bucket_prefix: prefix of the storage account, to which
'private' and 'public' will be appended
:param cache_path: path to the cache on the local file system
'''
self.credential: DefaultAzureCredential = DefaultAzureCredential()
self._blob_clients = {}
super().__init__(cache_path, cloud_type=CloudType.AZURE)
domain = 'blob.core.windows.net'
self.buckets: Dict[str:str] = {
StorageType.PRIVATE.value:
f'{bucket_prefix}{StorageType.PRIVATE.value}.{domain}',
StorageType.PUBLIC.value:
f'{bucket_prefix}{StorageType.PUBLIC.value}.{domain}'
}
# We pre-authenticate for the byoda container on each storage account
self.clients: Dict[StorageType, ContainerClient] = {
StorageType.PRIVATE.value:
ContainerClient(self.buckets[StorageType.PRIVATE.value],
'byoda',
credential=self.credential),
StorageType.PUBLIC.value:
ContainerClient(self.buckets[StorageType.PUBLIC.value],
'byoda',
credential=self.credential),
}
def __init__(
self,
account_url, # type: str
file_system_name, # type: str
credential=None, # type: Optional[Any]
**kwargs # type: Any
):
# type: (...) -> None
kwargs['retry_policy'] = kwargs.get(
'retry_policy') or ExponentialRetry(**kwargs)
super(FileSystemClient,
self).__init__(account_url,
file_system_name=file_system_name,
credential=credential,
**kwargs)
# to override the class field _container_client sync version
kwargs.pop('_hosts', None)
self._container_client = ContainerClient(
self._blob_account_url,
file_system_name,
credential=credential,
_hosts=self._container_client._hosts, # pylint: disable=protected-access
**kwargs) # type: ignore # pylint: disable=protected-access
self._client = DataLakeStorageClient(self.url,
file_system_name,
None,
pipeline=self._pipeline)
self._loop = kwargs.get('loop', None)
async def _test_container_sample_async(self):
# [START create_container_client_from_service]
# Instantiate a BlobServiceClient using a connection string
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(
self.connection_string)
# Instantiate a ContainerClient
container_client = blob_service_client.get_container_client(
"mynewcontainerasync")
# [END create_container_client_from_service]
# [START create_container_client_sasurl]
from azure.storage.blob.aio import ContainerClient
sas_url = sas_url = "https://account.blob.core.windows.net/mycontainer?sv=2015-04-05&st=2015-04-29T22%3A18%3A26Z&se=2015-04-30T02%3A23%3A26Z&sr=b&sp=rw&sip=168.1.5.60-168.1.5.70&spr=https&sig=Z%2FRHIX5Xcg0Mq2rqI3OlWTjEg2tYkboXr1P9ZUXDtkk%3D"
container = ContainerClient(sas_url)
# [END create_container_client_sasurl]
try:
# [START create_container]
await container_client.create_container()
# [END create_container]
# [START get_container_properties]
properties = await container_client.get_container_properties()
# [END get_container_properties]
assert properties is not None
finally:
# [START delete_container]
await container_client.delete_container()
async def _test_sas_signature_is_scrubbed_off(self):
# Test can only run live
if TestMode.need_recording_file(self.test_mode):
return
await self._setup()
# Arrange
container = self.bsc.get_container_client(self.container_name)
token = container.generate_shared_access_signature(
permission=ContainerPermissions.READ,
expiry=datetime.utcnow() + timedelta(hours=1),
)
# parse out the signed signature
token_components = parse_qs(token)
signed_signature = quote(
token_components[QueryStringConstants.SIGNED_SIGNATURE][0])
sas_service = ContainerClient(container.url, credential=token)
# Act
with LogCaptured(self) as log_captured:
await sas_service.get_account_information(logging_enable=True)
log_as_str = log_captured.getvalue()
# Assert
# make sure the query parameter 'sig' is logged, but its value is not
self.assertTrue(
QueryStringConstants.SIGNED_SIGNATURE in log_as_str)
self.assertFalse(signed_signature in log_as_str)
def __init__(self, blob_account_url, container_name, *, credential=None, **kwargs):
# type(str, str, Optional[Any], Any) -> None
container_client = kwargs.pop("container_client", None)
self._container_client = container_client or ContainerClient(
blob_account_url, container_name, credential=credential, **kwargs
)
self._cached_blob_clients = defaultdict() # type: Dict[str, BlobClient]
async def create_target_container(self):
container_client = ContainerClient(self.storage_endpoint,
self.target_container_name,
self.storage_key)
async with container_client:
await container_client.create_container()
return self.generate_sas_url(self.target_container_name, "wl")
async def _test_container_access_policy_async(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Instantiate a BlobServiceClient using a connection string
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(
self.connection_string)
# Instantiate a ContainerClient
container_client = blob_service_client.get_container_client(
"myaccesscontainerasync")
try:
# Create new Container
await container_client.create_container()
# [START set_container_access_policy]
# Create access policy
from azure.storage.blob import AccessPolicy, ContainerPermissions
access_policy = AccessPolicy(
permission=ContainerPermissions(read=True),
expiry=datetime.utcnow() + timedelta(hours=1),
start=datetime.utcnow() - timedelta(minutes=1))
identifiers = {'test': access_policy}
# Set the access policy on the container
await container_client.set_container_access_policy(
signed_identifiers=identifiers)
# [END set_container_access_policy]
# [START get_container_access_policy]
policy = await container_client.get_container_access_policy()
# [END get_container_access_policy]
# [START generate_sas_token]
# Use access policy to generate a sas token
sas_token = container_client.generate_shared_access_signature(
policy_id='my-access-policy-id')
# [END generate_sas_token]
# Use the sas token to authenticate a new client
# [START create_container_client_sastoken]
from azure.storage.blob.aio import ContainerClient
container = ContainerClient(
container_url=
"https://account.blob.core.windows.net/mycontainerasync",
credential=sas_token,
)
# [END create_container_client_sastoken]
finally:
# Delete container
await container_client.delete_container()
def test_container_client_api_version_property(self):
container_client = ContainerClient(
"https://foo.blob.core.windows.net/account",
self.container_name,
credential="fake_key")
self.assertEqual(container_client.api_version, self.api_version_2)
self.assertEqual(container_client._client._config.version, self.api_version_2)
container_client = ContainerClient(
"https://foo.blob.core.windows.net/account",
self.container_name,
credential="fake_key",
api_version=self.api_version_1)
self.assertEqual(container_client.api_version, self.api_version_1)
self.assertEqual(container_client._client._config.version, self.api_version_1)
blob_client = container_client.get_blob_client("foo")
self.assertEqual(blob_client.api_version, self.api_version_1)
self.assertEqual(blob_client._client._config.version, self.api_version_1)
def test_create_service_with_custom_account_endpoint_path(self):
account_name = "blobstorage"
account_key = "blobkey"
custom_account_url = "http://local-machine:11002/custom/account/path/" + self.sas_token
for service_type in SERVICES.items():
conn_string = 'DefaultEndpointsProtocol=http;AccountName={};AccountKey={};BlobEndpoint={};'.format(
account_name, account_key, custom_account_url)
# Act
service = service_type[0].from_connection_string(
conn_string, container_name="foo", blob_name="bar")
# Assert
self.assertEqual(service.account_name, account_name)
self.assertEqual(service.credential.account_name, account_name)
self.assertEqual(service.credential.account_key, account_key)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
service = BlobServiceClient(account_url=custom_account_url)
self.assertEqual(service.account_name, None)
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/?'))
service = ContainerClient(account_url=custom_account_url, container_name="foo")
self.assertEqual(service.account_name, None)
self.assertEqual(service.container_name, "foo")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/foo?'))
service = ContainerClient.from_container_url("http://local-machine:11002/custom/account/path/foo?query=value")
self.assertEqual(service.account_name, None)
self.assertEqual(service.container_name, "foo")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertEqual(service.url, 'http://local-machine:11002/custom/account/path/foo')
service = BlobClient(account_url=custom_account_url, container_name="foo", blob_name="bar", snapshot="baz")
self.assertEqual(service.account_name, None)
self.assertEqual(service.container_name, "foo")
self.assertEqual(service.blob_name, "bar")
self.assertEqual(service.snapshot, "baz")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/foo/bar?snapshot=baz&'))
service = BlobClient.from_blob_url("http://local-machine:11002/custom/account/path/foo/bar?snapshot=baz&query=value")
self.assertEqual(service.account_name, None)
self.assertEqual(service.container_name, "foo")
self.assertEqual(service.blob_name, "bar")
self.assertEqual(service.snapshot, "baz")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertEqual(service.url, 'http://local-machine:11002/custom/account/path/foo/bar?snapshot=baz')
async def create_source_container(self):
container_client = ContainerClient(self.storage_endpoint,
self.source_container_name,
self.storage_key)
async with container_client:
await container_client.create_container()
docs = Document.create_docs(10)
for blob in docs:
await container_client.upload_blob(name=blob.prefix +
blob.name + blob.suffix,
data=blob.data)
return self.generate_sas_url(self.source_container_name, "rl")
def test_invalid_api_version(self):
with pytest.raises(ValueError) as error:
BlobServiceClient("https://foo.blob.core.windows.net/account",
credential="fake_key",
api_version="foo")
self.assertTrue(
str(error.value).startswith("Unsupported API version 'foo'."))
with pytest.raises(ValueError) as error:
ContainerClient("https://foo.blob.core.windows.net/account",
self.container_name,
credential="fake_key",
api_version="foo")
self.assertTrue(
str(error.value).startswith("Unsupported API version 'foo'."))
with pytest.raises(ValueError) as error:
BlobClient("https://foo.blob.core.windows.net/account",
self.container_name,
self._get_blob_reference(),
credential="fake_key",
api_version="foo")
self.assertTrue(
str(error.value).startswith("Unsupported API version 'foo'."))
async def batch_translation_with_storage_async(self):
# import libraries
from azure.core.credentials import AzureKeyCredential
from azure.ai.documenttranslation.aio import DocumentTranslationClient
from azure.ai.documenttranslation import (DocumentTranslationInput,
TranslationTarget)
from azure.storage.blob.aio import ContainerClient
from azure.storage.blob import (generate_container_sas,
ContainerSasPermissions)
# get service secrets
endpoint = os.environ["AZURE_DOCUMENT_TRANSLATION_ENDPOINT"]
key = os.environ["AZURE_DOCUMENT_TRANSLATION_KEY"]
source_storage_endpoint = os.environ["AZURE_STORAGE_SOURCE_ENDPOINT"]
source_storage_account_name = os.environ[
"AZURE_STORAGE_SOURCE_ACCOUNT_NAME"]
source_storage_container_name = os.environ[
"AZURE_STORAGE_SOURCE_CONTAINER_NAME"]
source_storage_key = os.environ["AZURE_STORAGE_SOURCE_KEY"]
target_storage_endpoint = os.environ["AZURE_STORAGE_TARGET_ENDPOINT"]
target_storage_account_name = os.environ[
"AZURE_STORAGE_TARGET_ACCOUNT_NAME"]
target_storage_container_name = os.environ[
"AZURE_STORAGE_TARGET_CONTAINER_NAME"]
target_storage_key = os.environ["AZURE_STORAGE_TARGET_KEY"]
# create service clients
translation_client = DocumentTranslationClient(endpoint,
AzureKeyCredential(key))
container_client = ContainerClient(
source_storage_endpoint,
container_name=source_storage_container_name,
credential=source_storage_key)
# upload some document for translation
with open("document.txt", "rb") as doc:
await container_client.upload_blob(name="document.txt", data=doc)
# prepare translation job input
source_container_sas = generate_container_sas(
account_name=source_storage_account_name,
container_name=source_storage_container_name,
account_key=source_storage_key,
permission=ContainerSasPermissions.from_string("rl"))
target_container_sas = generate_container_sas(
account_name=target_storage_account_name,
container_name=target_storage_container_name,
account_key=target_storage_key,
permission=ContainerSasPermissions.from_string("rlwd"))
source_container_url = source_storage_endpoint + "/" + source_storage_container_name + "?" + source_container_sas
target_container_url = target_storage_endpoint + "/" + target_storage_container_name + "?" + target_container_sas
translation_inputs = [
DocumentTranslationInput(source_url=source_container_url,
targets=[
TranslationTarget(
target_url=target_container_url,
language_code="es")
],
prefix="document")
]
# run job
async with translation_client:
job_detail = await translation_client.create_translation_job(
translation_inputs)
job_result = await translation_client.wait_until_done(job_detail.id
)
# poll status result
if job_result.status == "Succeeded":
print("We translated our documents!")
if job_result.documents_failed_count > 0:
await self.check_documents(translation_client,
job_result.id)
elif job_result.status in ["Failed", "ValidationFailed"]:
if job_result.error:
print("Translation job failed: {}: {}".format(
job_result.error.code, job_result.error.message))
await self.check_documents(translation_client, job_result.id)
exit(1)
# store result documents
container_client = ContainerClient(
target_storage_endpoint,
container_name=target_storage_container_name,
credential=target_storage_key)
with open("translated.txt", "wb") as my_blob:
download_stream = await container_client.download_blob(
"document.txt")
my_blob.write(await download_stream.readall())