async def container_access_policy_async(self):
# Instantiate a BlobServiceClient using a connection string
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(
self.connection_string)
async with blob_service_client:
# Instantiate a ContainerClient
container_client = blob_service_client.get_container_client(
"myaccesscontainerasync")
try:
# Create new Container
await container_client.create_container()
# [START set_container_access_policy]
# Create access policy
from azure.storage.blob import AccessPolicy, ContainerSasPermissions
access_policy = AccessPolicy(
permission=ContainerSasPermissions(read=True),
expiry=datetime.utcnow() + timedelta(hours=1),
start=datetime.utcnow() - timedelta(minutes=1))
identifiers = {'my-access-policy-id': access_policy}
# Set the access policy on the container
await container_client.set_container_access_policy(
signed_identifiers=identifiers)
# [END set_container_access_policy]
# [START get_container_access_policy]
policy = await container_client.get_container_access_policy()
# [END get_container_access_policy]
# [START generate_sas_token]
# Use access policy to generate a sas token
from azure.storage.blob import generate_container_sas
sas_token = generate_container_sas(
container_client.account_name,
container_client.container_name,
account_key=container_client.credential.account_key,
policy_id='my-access-policy-id')
# [END generate_sas_token]
# Use the sas token to authenticate a new client
# [START create_container_client_sastoken]
from azure.storage.blob.aio import ContainerClient
container = ContainerClient.from_container_url(
container_url=
"https://account.blob.core.windows.net/mycontainerasync",
credential=sas_token,
)
# [END create_container_client_sastoken]
finally:
# Delete container
await container_client.delete_container()
async def main():
container_client = ContainerClient.from_connection_string(STORAGE_CONNECTION_STR, "eventprocessor")
partition_manager = BlobPartitionManager(container_client)
client = EventHubConsumerClient.from_connection_string(
CONNECTION_STR,
partition_manager=partition_manager, # For load balancing and checkpoint. Leave None for no load balancing
)
async with client:
await receive(client)
async def _test_container_access_policy_async(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Instantiate a BlobServiceClient using a connection string
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(
self.connection_string)
# Instantiate a ContainerClient
container_client = blob_service_client.get_container_client(
"myaccesscontainerasync")
try:
# Create new Container
await container_client.create_container()
# [START set_container_access_policy]
# Create access policy
from azure.storage.blob import AccessPolicy, ContainerSasPermissions
access_policy = AccessPolicy(
permission=ContainerSasPermissions(read=True),
expiry=datetime.utcnow() + timedelta(hours=1),
start=datetime.utcnow() - timedelta(minutes=1))
identifiers = {'test': access_policy}
# Set the access policy on the container
await container_client.set_container_access_policy(
signed_identifiers=identifiers)
# [END set_container_access_policy]
# [START get_container_access_policy]
policy = await container_client.get_container_access_policy()
# [END get_container_access_policy]
# [START generate_sas_token]
# Use access policy to generate a sas token
sas_token = container_client.generate_shared_access_signature(
policy_id='my-access-policy-id')
# [END generate_sas_token]
# Use the sas token to authenticate a new client
# [START create_container_client_sastoken]
from azure.storage.blob.aio import ContainerClient
container = ContainerClient.from_container_url(
container_url=
"https://account.blob.core.windows.net/mycontainerasync",
credential=sas_token,
)
# [END create_container_client_sastoken]
finally:
# Delete container
await container_client.delete_container()
def test_container_client_api_version_property(self):
container_client = ContainerClient(
"https://foo.blob.core.windows.net/account",
self.container_name,
credential="fake_key")
self.assertEqual(container_client.api_version, self.api_version_2)
self.assertEqual(container_client._client._config.version, self.api_version_2)
container_client = ContainerClient(
"https://foo.blob.core.windows.net/account",
self.container_name,
credential="fake_key",
api_version=self.api_version_1)
self.assertEqual(container_client.api_version, self.api_version_1)
self.assertEqual(container_client._client._config.version, self.api_version_1)
blob_client = container_client.get_blob_client("foo")
self.assertEqual(blob_client.api_version, self.api_version_1)
self.assertEqual(blob_client._client._config.version, self.api_version_1)
def __enter__(self):
self._client = EventHubClient.from_connection_string(
self._eh_connection_string)
self._storage_client = ContainerClient.from_connection_string(
conn_str=self._sa_connection_string,
container=self._sa_container_name)
self._partition_manager = BlobPartitionManager(self._storage_client)
self._event_processor = EventProcessor(self._client, "$default",
self._partition_processor,
self._partition_manager)
async def generate_append_blob(
container_client: ContainerClient,
manager: ServiceManagerBase,
store_param: str,
time_based=False,
):
"""
Generates an append blob, either based on a random UUID or the current time (hour and minute).
:param container_client: Container Client.
:param manager: The manager calling for the new file name generation.
:param store_param: Name of the folder the persistor will store the data to.
:param time_based: Whether or not the blob's name should be generated based on the current time.
:return: Complete file path to store the blob to.
"""
now = datetime.now()
# If time-based, set the file name based on time (hour/minute).
if time_based:
name = "{hour}-{minute}".format(hour=str(now.hour),
minute=str(now.minute))
# Otherwise, generate the name based on a random UUID
else:
name = str(uuid.uuid4())
file_name = generate_file_name(store_param, name)
# To help minimize concurrency issues, we introduce an async lock within the same instance.
# (Not much we can do with concurrent instances/workers.)
async with manager.append_lock:
if not time_based or not manager.now or now.hour > manager.now.hour or now.minute > manager.now.minute:
blob_client = container_client.get_blob_client(file_name)
# Try to get the blob properties. If the blob doesn't exist, this will return an exception.
try:
await blob_client.get_blob_properties()
except ResourceNotFoundError:
# Since the blob doesn't appear to exist, try to create a new one.
# Since we need to think about concurrency between different instances that
# don't share the manager object,
# we have to assume some exception will occur when multiple creation attempts are made.
try:
await blob_client.create_append_blob()
# Unsure which exception might be thrown if concurrently attempting to create the append blob.
except:
pass
manager.now = now
manager.file_name = file_name
return file_name
async def create_source_container(self):
container_client = ContainerClient(self.storage_endpoint,
self.source_container_name,
self.storage_key)
async with container_client:
await container_client.create_container()
docs = Document.create_docs(10)
for blob in docs:
await container_client.upload_blob(name=blob.prefix +
blob.name + blob.suffix,
data=blob.data)
return self.generate_sas_url(self.source_container_name, "rl")
async def main():
try:
CONNECTION_STRING = os.environ['AZURE_STORAGE_CONNECTION_STRING']
except KeyError:
print("AZURE_STORAGE_CONNECTION_STRING must be set.")
sys.exit(1)
container = ContainerClient.from_connection_string(
CONNECTION_STRING, container_name="mycontainer")
async for blob in container.list_blobs():
print(blob.name + '\n')
async def main():
try:
CONNECTION_STRING = "xxxx"
except KeyError:
print("AZURE_STORAGE_CONNECTION_STRING must be set.")
sys.exit(1)
container = ContainerClient.from_connection_string(CONNECTION_STRING, container_name="containerName")
path="example/data/"
async with container:
async for blob in container.list_blobs(name_starts_with=path):
print(blob.name + '\n')
def __init__(self):
connection_string = os.environ.get("STORAGE_CONNECTION_STRING")
if not connection_string:
raise Exception(
"Undefined environment variable STORAGE_CONNECTION_STRING")
self.container_client = SyncContainerClient.from_connection_string(
conn_str=connection_string, container_name=self.container_name)
#TODO: I really hate this.
self.async_container_client = AsyncContainerClient.from_connection_string(
conn_str=connection_string, container_name=self.container_name)
async def main():
eventhub_client = EventHubClient.from_connection_string(
EVENT_HUB_CONNECTION_STR, receive_timeout=5, retry_total=3)
storage_container_client = ContainerClient.from_connection_string(
STORAGE_CONTAINER_CONNECTION_STR, STORAGE_CONTAINER_PATH)
partition_manager = BlobPartitionManager(
storage_container_client) # use the BlobPartitonManager to save
event_processor = EventProcessor(eventhub_client, "mllab-consumers-group",
MyPartitionProcessor, partition_manager)
print("Starting...")
async with storage_container_client:
asyncio.ensure_future(event_processor.start())
await asyncio.sleep(20) # run for a while
await event_processor.stop()
def __init__(
self,
container: ContainerClient,
blob_name: str,
content_type: str,
content_encoding: str,
):
self.container = container
self.blob_name = blob_name
self.content_settings = ContentSettings(
content_type=content_type, content_encoding=content_encoding)
self.state = StreamedBlobState.not_initialized
self.blob = container.get_blob_client(blob_name)
self.blocks = []
async def get_async_container_clients(self):
# This is equivalent to:
# client1 = AsyncContainerClient.from_container_url(url1)
# ...
# client16 = AsyncContainerClient.from_container_url(url16)
# async with client1, ..., client16:
# yield {prefix1: client1, ..., prefix16: client16}
clients = {
prefix: AsyncContainerClient.from_container_url(url)
for (prefix, url) in self.container_urls.items()
}
async with contextlib.AsyncExitStack() as stack:
for client in clients.values():
await stack.enter_async_context(client)
yield clients
def download(paths):
now = lambda: time.time()
start = now()
tasks = []
for path in paths:
container_client = ContainerClient.from_connection_string(
conn_str=os.getenv("AZURE_STORAGE_CONNECTION_STRING"),
container_name=os.getenv("CONTAINER_NAME"))
tasks.append(downloadPath(container_client, path))
if tasks:
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))
print('TIME: ', now() - start)
async def get_blob_service_client_from_container_client_async(self):
# Instantiate a BlobServiceClient using a connection string
from azure.storage.blob.aio import ContainerClient
container_client1 = ContainerClient.from_connection_string(
self.connection_string, "container")
await container_client1.create_container()
# [START get_blob_service_client_from_container_client]
blob_service_client = container_client1.get_blob_service_client()
print(await blob_service_client.get_service_properties())
container_client2 = blob_service_client.get_container_client(
"container")
print(await container_client2.get_container_properties())
await container_client2.delete_container()
await container_client1.close()
def get_live_storage_blob_client():
try:
storage_connection_str = os.environ['AZURE_STORAGE_CONN_STR']
except KeyError:
return None, None
try:
from azure.storage.blob import BlobServiceClient
from azure.storage.blob.aio import ContainerClient
except ImportError or ModuleNotFoundError:
return None, None
container_str = str(uuid.uuid4())
blob_service_client = BlobServiceClient.from_connection_string(
storage_connection_str)
blob_service_client.create_container(container_str)
container_client = ContainerClient.from_connection_string(
storage_connection_str, container_str)
return container_str, container_client
async def init_blob_for_streaming_upload(
container: ContainerClient,
blob_name: str,
content_type: str,
content_encoding: str,
data: Any,
return_sas_token: bool = True,
) -> str:
"""
Uploads the given data to a blob record.
If a blob with the given name already exist, it throws an error.
Returns a uri with a SAS token to access the newly created blob.
"""
await create_container_using_client(container)
logger.info(f"Streaming blob '{blob_name}'" +
f"to container '{container.container_name}' on account:" +
f"'{container.account_name}'")
content_settings = ContentSettings(content_type=content_type,
content_encoding=content_encoding)
blob = container.get_blob_client(blob_name)
await blob.stage_block()
await blob.commit_block_list()
await blob.upload_blob(data, content_settings=content_settings)
logger.debug(f" - blob '{blob_name}' uploaded. generating sas token.")
if return_sas_token:
sas_token = generate_blob_sas(
blob.account_name,
blob.container_name,
blob.blob_name,
account_key=blob.credential.account_key,
permission=BlobSasPermissions(read=True),
expiry=datetime.utcnow() + timedelta(days=14),
)
uri = blob.url + "?" + sas_token
else:
uri = remove_sas_token(blob.url)
logger.debug(f" - blob access url: '{uri}'.")
return uri
async def append_blob(
container: ContainerClient,
blob_name: str,
content_type: str,
content_encoding: str,
data: Any,
return_sas_token: bool = True,
metadata: Dict[str, str] = None,
) -> str:
"""
Uploads the given data to a blob record.
If a blob with the given name already exist, it throws an error.
Returns a uri with a SAS token to access the newly created blob.
"""
await create_container_using_client(container)
logger.info(f"Appending data to blob '{blob_name}'" +
f"in container '{container.container_name}'" +
f"on account: '{container.account_name}'")
content_settings = ContentSettings(content_type=content_type,
content_encoding=content_encoding)
blob = container.get_blob_client(blob_name)
try:
props = await blob.get_blob_properties()
if props.blob_type != BlobType.AppendBlob:
raise Exception("blob must be an append blob")
except exceptions.ResourceNotFoundError:
props = await blob.create_append_blob(
content_settings=content_settings, metadata=metadata)
await blob.append_block(data, len(data))
logger.debug(f" - blob '{blob_name}' appended. generating sas token.")
if return_sas_token:
uri = get_blob_uri_with_sas_token(blob)
else:
uri = remove_sas_token(blob.url)
logger.debug(f" - blob access url: '{uri}'.")
return uri
def __init__(
self, account_url, # type: str
file_system_name, # type: str
credential=None, # type: Optional[Any]
**kwargs # type: Any
):
# type: (...) -> None
kwargs['retry_policy'] = kwargs.get('retry_policy') or ExponentialRetry(**kwargs)
super(FileSystemClient, self).__init__(
account_url,
file_system_name=file_system_name,
credential=credential,
**kwargs)
# to override the class field _container_client sync version
kwargs.pop('_hosts', None)
self._container_client = ContainerClient(self._blob_account_url, file_system_name,
credential=credential,
_hosts=self._container_client._hosts,# pylint: disable=protected-access
**kwargs) # type: ignore # pylint: disable=protected-access
self._client = DataLakeStorageClient(self.url, file_system_name, None, pipeline=self._pipeline)
self._loop = kwargs.get('loop', None)
async def _test_auth_connection_string_async(self):
# [START auth_from_connection_string]
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(self.connection_string)
# [END auth_from_connection_string]
# [START auth_from_connection_string_container]
from azure.storage.blob.aio import ContainerClient
container_client = ContainerClient.from_connection_string(
self.connection_string, container_name="mycontainer")
# [END auth_from_connection_string_container]
# [START auth_from_connection_string_blob]
from azure.storage.blob.aio import BlobClient
blob_client = BlobClient.from_connection_string(
self.connection_string, container_name="mycontainer", blob_name="blobname.txt")
# [END auth_from_connection_string_blob]
# Get account information for the Blob Service
account_info = await blob_service_client.get_account_information()
assert account_info is not None
async def GlobalSetupAsync(self):
connection_string = os.environ.get("STORAGE_CONNECTION_STRING")
if not connection_string:
raise Exception(
"Undefined environment variable STORAGE_CONNECTION_STRING")
type(self
).container_client = SyncContainerClient.from_connection_string(
conn_str=connection_string,
container_name=self.container_name)
type(
self
).async_container_client = AsyncContainerClient.from_connection_string(
conn_str=connection_string, container_name=self.container_name)
await type(self).async_container_client.__aenter__()
type(self).container_client.create_container()
data = b'a' * self.Arguments.size
type(self).container_client.upload_blob(self.blob_name, data)
async def _upload_blob() -> None:
async with ContainerClient.from_connection_string( # type: ignore[attr-defined]
_connection_string(subscription, resource_group, storage_name),
container_name,
) as container_client:
class UploadTasks:
def __init__(self,
paths_to_upload: List[pathlib.Path],
max_open_files: int = 100):
self.index = 0
self.paths_to_upload = paths_to_upload
self.semaphore = asyncio.Semaphore(max_open_files)
# Forward reference not possible on Python 3.6:
def __iter__(self): # type: ignore[no-untyped-def]
return self
def __next__(self) -> asyncio.Task:
try:
task = asyncio.create_task(
_upload_file(
container_client,
self.paths_to_upload[self.index],
source_folder,
self.semaphore,
))
except IndexError as exc:
raise StopIteration from exc
self.index += 1
return task
def __len__(self) -> int:
return len(self.paths_to_upload)
for task in tqdm.as_completed(
UploadTasks(paths_to_upload),
bar_format="{l_bar} {bar} | Uploaded {n_fmt}/{total_fmt}",
):
await task
def from_connection_string(
cls, conn_str, container_name, *, credential=None, **kwargs
):
# type: (str, str, Optional[Any], str) -> BlobCheckpointStore
"""Create BlobCheckpointStore from a storage connection string.
:param str conn_str:
A connection string to an Azure Storage account.
:param container_name:
The container name for the blobs.
:type container_name: str
:param credential:
The credentials with which to authenticate. This is optional if the
account URL already has a SAS token, or the connection string already has shared
access key values. The value can be a SAS token string, an account shared access
key, or an instance of a TokenCredentials class from azure.identity.
Credentials provided here will take precedence over those in the connection string.
"""
container_client = ContainerClient.from_connection_string(
conn_str, container_name, credential=credential, **kwargs
)
return cls(None, None, container_client=container_client)
async def test_sas_signature_is_scrubbed_off(self, resource_group,
location, storage_account,
storage_account_key):
# Test can only run live
if not self.is_live:
pytest.skip("live only")
bsc = BlobServiceClient(self._account_url(storage_account.name),
storage_account_key)
await self._setup(bsc)
# Arrange
container = bsc.get_container_client(self.container_name)
token = generate_container_sas(
container.account_name,
container.container_name,
account_key=container.credential.account_key,
permission=ContainerSasPermissions(read=True),
expiry=datetime.utcnow() + timedelta(hours=1),
)
# parse out the signed signature
token_components = parse_qs(token)
signed_signature = quote(
token_components[QueryStringConstants.SIGNED_SIGNATURE][0])
sas_service = ContainerClient.from_container_url(container.url,
credential=token)
# Act
with LogCaptured(self) as log_captured:
await sas_service.get_account_information(logging_enable=True)
log_as_str = log_captured.getvalue()
# Assert
# make sure the query parameter 'sig' is logged, but its value is not
self.assertTrue(
QueryStringConstants.SIGNED_SIGNATURE in log_as_str)
self.assertFalse(signed_signature in log_as_str)
def test_invalid_api_version(self):
with pytest.raises(ValueError) as error:
BlobServiceClient("https://foo.blob.core.windows.net/account",
credential="fake_key",
api_version="foo")
self.assertTrue(
str(error.value).startswith("Unsupported API version 'foo'."))
with pytest.raises(ValueError) as error:
ContainerClient("https://foo.blob.core.windows.net/account",
self.container_name,
credential="fake_key",
api_version="foo")
self.assertTrue(
str(error.value).startswith("Unsupported API version 'foo'."))
with pytest.raises(ValueError) as error:
BlobClient("https://foo.blob.core.windows.net/account",
self.container_name,
self._get_blob_reference(),
credential="fake_key",
api_version="foo")
self.assertTrue(
str(error.value).startswith("Unsupported API version 'foo'."))
def _create_container_client(self):
return ContainerClient.from_connection_string(self.__conn_string,
self.__container_name,
logging_enable=False)
async def save_to_storage(
data: List[str],
container_client: ContainerClient,
store_param: str,
append=False,
file_append_name: Optional[str] = None,
):
"""
Saves message to storage.
:param data: List of data be stored.
:param container_client: Blob service client (initialized outside this function).
:param store_param: The main folder in the container to store the file in.
:param append: Flag to determine whether the data should be appended to an append blob.
:param file_append_name: Name of the append blob to store to. Ignored if append is False.
:return: Name of the blob stored to and result
"""
# Success flag.
result = False
# Get the blob file name and the data (string) to store.
if not append:
file_name = generate_file_name(store_param=store_param)
else:
file_name = None
# If the file_name is None, we should be using the append blob name.
# If the append blob name is not given, an exception is raised.
if not file_name:
if not file_append_name:
raise StorageTypeConfigurationException(
"SET BLOB TO 'APPEND', YET NO FILE GIVEN FOR THE APPEND BLOB!")
file_name = file_append_name
# Store the data utilizing a simple retry policy.
# In truth, the Blob Client we're using already uses an ExponentialRetry mechanic. This is
# merely an additional fail-safe to it, in case the library at some point changes some of
# the default retry parameters or the save load per second is far bigger than expected.
# In addition, on the off-chance the user is using the TIMED_APPEND option, this retry loop helps with
# potential concurrency issues. If the function manages to get to this point without an append blob
# existing, this loop will give enough time for it to be created in the meantime and ensure a successful
# store.
# In practice, this loop will not execute more than once.
for i in range(STORE_RETRY_POLICY_MAX):
try:
# We include getting the blob client in the retry, due to the fact we likely
# need to renew the lease for the blob.
blob_client = container_client.get_blob_client(file_name, )
if append:
store_method = blob_client.append_block
else:
store_method = blob_client.upload_blob
async with blob_client:
await store_method("\n".join(data))
# Set the result to true.
result = True
# Escape the retry loop.
break
# Currently set to catch a general exception, seeing as how the documentation doesn't
# actually state the possible exceptions that could occur during these processes.
except Exception as exc:
if i == STORE_RETRY_POLICY_MAX - 1:
logging.error(
"FAILED TO SAVE TO STORAGE! | PATH: %s | FAILED MESSAGE CONTENTS: %s | EXCEPTION %s",
file_name,
data,
str(exc),
)
else:
logging.warning(
"FAILED TO SAVE TO STORAGE! | PATH: %s | RETRYING... (ATTEMPT NO. %s)",
file_name,
str(i + 1),
)
await asyncio.sleep(STORE_RETRY_BACKOFF_TIME)
return file_name, result
async def batch_translation_with_storage_async(self):
# import libraries
from azure.core.credentials import AzureKeyCredential
from azure.ai.documenttranslation.aio import DocumentTranslationClient
from azure.ai.documenttranslation import (DocumentTranslationInput,
TranslationTarget)
from azure.storage.blob.aio import ContainerClient
from azure.storage.blob import (generate_container_sas,
ContainerSasPermissions)
# get service secrets
endpoint = os.environ["AZURE_DOCUMENT_TRANSLATION_ENDPOINT"]
key = os.environ["AZURE_DOCUMENT_TRANSLATION_KEY"]
source_storage_endpoint = os.environ["AZURE_STORAGE_SOURCE_ENDPOINT"]
source_storage_account_name = os.environ[
"AZURE_STORAGE_SOURCE_ACCOUNT_NAME"]
source_storage_container_name = os.environ[
"AZURE_STORAGE_SOURCE_CONTAINER_NAME"]
source_storage_key = os.environ["AZURE_STORAGE_SOURCE_KEY"]
target_storage_endpoint = os.environ["AZURE_STORAGE_TARGET_ENDPOINT"]
target_storage_account_name = os.environ[
"AZURE_STORAGE_TARGET_ACCOUNT_NAME"]
target_storage_container_name = os.environ[
"AZURE_STORAGE_TARGET_CONTAINER_NAME"]
target_storage_key = os.environ["AZURE_STORAGE_TARGET_KEY"]
# create service clients
translation_client = DocumentTranslationClient(endpoint,
AzureKeyCredential(key))
container_client = ContainerClient(
source_storage_endpoint,
container_name=source_storage_container_name,
credential=source_storage_key)
# upload some document for translation
with open("document.txt", "rb") as doc:
await container_client.upload_blob(name="document.txt", data=doc)
# prepare translation job input
source_container_sas = generate_container_sas(
account_name=source_storage_account_name,
container_name=source_storage_container_name,
account_key=source_storage_key,
permission=ContainerSasPermissions.from_string("rl"))
target_container_sas = generate_container_sas(
account_name=target_storage_account_name,
container_name=target_storage_container_name,
account_key=target_storage_key,
permission=ContainerSasPermissions.from_string("rlwd"))
source_container_url = source_storage_endpoint + "/" + source_storage_container_name + "?" + source_container_sas
target_container_url = target_storage_endpoint + "/" + target_storage_container_name + "?" + target_container_sas
translation_inputs = [
DocumentTranslationInput(source_url=source_container_url,
targets=[
TranslationTarget(
target_url=target_container_url,
language_code="es")
],
prefix="document")
]
# run job
async with translation_client:
job_detail = await translation_client.create_translation_job(
translation_inputs)
job_result = await translation_client.wait_until_done(job_detail.id
)
# poll status result
if job_result.status == "Succeeded":
print("We translated our documents!")
if job_result.documents_failed_count > 0:
await self.check_documents(translation_client,
job_result.id)
elif job_result.status in ["Failed", "ValidationFailed"]:
if job_result.error:
print("Translation job failed: {}: {}".format(
job_result.error.code, job_result.error.message))
await self.check_documents(translation_client, job_result.id)
exit(1)
# store result documents
container_client = ContainerClient(
target_storage_endpoint,
container_name=target_storage_container_name,
credential=target_storage_key)
with open("translated.txt", "wb") as my_blob:
download_stream = await container_client.download_blob(
"document.txt")
my_blob.write(await download_stream.readall())
STORAGE_CONNECTION_STR = os.environ["AZURE_STORAGE_CONN_STR"]
logging.basicConfig(level=logging.INFO)
async def do_operation(event):
# put your code here
# do some sync or async operations. If the operation is i/o intensive, async will have better performance
print(event)
async def process_events(partition_context, events):
# put your code here
await asyncio.gather(*[do_operation(event) for event in events])
await partition_context.update_checkpoint(events[-1])
if __name__ == '__main__':
loop = asyncio.get_event_loop()
container_client = ContainerClient.from_connection_string(
STORAGE_CONNECTION_STR, "eventprocessor")
partition_manager = BlobPartitionManager(container_client=container_client)
client = EventHubConsumerClient.from_connection_string(
CONNECTION_STR, partition_manager=partition_manager)
try:
loop.run_until_complete(client.receive(process_events, "$default"))
except KeyboardInterrupt:
loop.run_until_complete(client.close())
finally:
loop.stop()
def test_create_service_with_custom_account_endpoint_path(self):
account_name = "blobstorage"
account_key = "blobkey"
custom_account_url = "http://local-machine:11002/custom/account/path/" + self.sas_token
for service_type in SERVICES.items():
conn_string = 'DefaultEndpointsProtocol=http;AccountName={};AccountKey={};BlobEndpoint={};'.format(
account_name, account_key, custom_account_url)
# Act
service = service_type[0].from_connection_string(
conn_string, container_name="foo", blob_name="bar")
# Assert
self.assertEqual(service.account_name, account_name)
self.assertEqual(service.credential.account_name, account_name)
self.assertEqual(service.credential.account_key, account_key)
self.assertEqual(service.primary_hostname,
'local-machine:11002/custom/account/path')
service = BlobServiceClient(account_url=custom_account_url)
self.assertEqual(service.account_name, None)
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname,
'local-machine:11002/custom/account/path')
self.assertTrue(
service.url.startswith(
'http://local-machine:11002/custom/account/path/?'))
service = ContainerClient(account_url=custom_account_url,
container_name="foo")
self.assertEqual(service.account_name, None)
self.assertEqual(service.container_name, "foo")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname,
'local-machine:11002/custom/account/path')
self.assertTrue(
service.url.startswith(
'http://local-machine:11002/custom/account/path/foo?'))
service = ContainerClient.from_container_url(
"http://local-machine:11002/custom/account/path/foo?query=value")
self.assertEqual(service.account_name, None)
self.assertEqual(service.container_name, "foo")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname,
'local-machine:11002/custom/account/path')
self.assertEqual(service.url,
'http://local-machine:11002/custom/account/path/foo')
service = BlobClient(account_url=custom_account_url,
container_name="foo",
blob_name="bar",
snapshot="baz")
self.assertEqual(service.account_name, None)
self.assertEqual(service.container_name, "foo")
self.assertEqual(service.blob_name, "bar")
self.assertEqual(service.snapshot, "baz")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname,
'local-machine:11002/custom/account/path')
self.assertTrue(
service.url.startswith(
'http://local-machine:11002/custom/account/path/foo/bar?snapshot=baz&'
))
service = BlobClient.from_blob_url(
"http://local-machine:11002/custom/account/path/foo/bar?snapshot=baz&query=value"
)
self.assertEqual(service.account_name, None)
self.assertEqual(service.container_name, "foo")
self.assertEqual(service.blob_name, "bar")
self.assertEqual(service.snapshot, "baz")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname,
'local-machine:11002/custom/account/path')
self.assertEqual(
service.url,
'http://local-machine:11002/custom/account/path/foo/bar?snapshot=baz'
)
