async def upload_blob(
container: ContainerClient,
blob_name: str,
content_type: str,
content_encoding: str,
data: Any,
return_sas_token: bool = True,
) -> str:
"""
Uploads the given data to a blob record.
If a blob with the given name already exist, it throws an error.
Returns a uri with a SAS token to access the newly created blob.
"""
await create_container_using_client(container)
logger.info(f"Uploading blob '{blob_name}'" +
f"to container '{container.container_name}'" +
f"on account: '{container.account_name}'")
content_settings = ContentSettings(content_type=content_type,
content_encoding=content_encoding)
blob = container.get_blob_client(blob_name)
await blob.upload_blob(data, content_settings=content_settings)
logger.debug(f" - blob '{blob_name}' uploaded. generating sas token.")
if return_sas_token:
uri = get_blob_uri_with_sas_token(blob)
else:
uri = remove_sas_token(blob.url)
logger.debug(f" - blob access url: '{uri}'.")
return uri
async def generate_append_blob(
container_client: ContainerClient,
manager: ServiceManagerBase,
store_param: str,
time_based=False,
):
"""
Generates an append blob, either based on a random UUID or the current time (hour and minute).
:param container_client: Container Client.
:param manager: The manager calling for the new file name generation.
:param store_param: Name of the folder the persistor will store the data to.
:param time_based: Whether or not the blob's name should be generated based on the current time.
:return: Complete file path to store the blob to.
"""
now = datetime.now()
# If time-based, set the file name based on time (hour/minute).
if time_based:
name = "{hour}-{minute}".format(hour=str(now.hour),
minute=str(now.minute))
# Otherwise, generate the name based on a random UUID
else:
name = str(uuid.uuid4())
file_name = generate_file_name(store_param, name)
# To help minimize concurrency issues, we introduce an async lock within the same instance.
# (Not much we can do with concurrent instances/workers.)
async with manager.append_lock:
if not time_based or not manager.now or now.hour > manager.now.hour or now.minute > manager.now.minute:
blob_client = container_client.get_blob_client(file_name)
# Try to get the blob properties. If the blob doesn't exist, this will return an exception.
try:
await blob_client.get_blob_properties()
except ResourceNotFoundError:
# Since the blob doesn't appear to exist, try to create a new one.
# Since we need to think about concurrency between different instances that
# don't share the manager object,
# we have to assume some exception will occur when multiple creation attempts are made.
try:
await blob_client.create_append_blob()
# Unsure which exception might be thrown if concurrently attempting to create the append blob.
except:
pass
manager.now = now
manager.file_name = file_name
return file_name
def __init__(
self,
container: ContainerClient,
blob_name: str,
content_type: str,
content_encoding: str,
):
self.container = container
self.blob_name = blob_name
self.content_settings = ContentSettings(
content_type=content_type, content_encoding=content_encoding)
self.state = StreamedBlobState.not_initialized
self.blob = container.get_blob_client(blob_name)
self.blocks = []
async def init_blob_for_streaming_upload(
container: ContainerClient,
blob_name: str,
content_type: str,
content_encoding: str,
data: Any,
return_sas_token: bool = True,
) -> str:
"""
Uploads the given data to a blob record.
If a blob with the given name already exist, it throws an error.
Returns a uri with a SAS token to access the newly created blob.
"""
await create_container_using_client(container)
logger.info(f"Streaming blob '{blob_name}'" +
f"to container '{container.container_name}' on account:" +
f"'{container.account_name}'")
content_settings = ContentSettings(content_type=content_type,
content_encoding=content_encoding)
blob = container.get_blob_client(blob_name)
await blob.stage_block()
await blob.commit_block_list()
await blob.upload_blob(data, content_settings=content_settings)
logger.debug(f" - blob '{blob_name}' uploaded. generating sas token.")
if return_sas_token:
sas_token = generate_blob_sas(
blob.account_name,
blob.container_name,
blob.blob_name,
account_key=blob.credential.account_key,
permission=BlobSasPermissions(read=True),
expiry=datetime.utcnow() + timedelta(days=14),
)
uri = blob.url + "?" + sas_token
else:
uri = remove_sas_token(blob.url)
logger.debug(f" - blob access url: '{uri}'.")
return uri
def test_container_client_api_version_property(self):
container_client = ContainerClient(
"https://foo.blob.core.windows.net/account",
self.container_name,
credential="fake_key")
self.assertEqual(container_client.api_version, self.api_version_2)
self.assertEqual(container_client._client._config.version, self.api_version_2)
container_client = ContainerClient(
"https://foo.blob.core.windows.net/account",
self.container_name,
credential="fake_key",
api_version=self.api_version_1)
self.assertEqual(container_client.api_version, self.api_version_1)
self.assertEqual(container_client._client._config.version, self.api_version_1)
blob_client = container_client.get_blob_client("foo")
self.assertEqual(blob_client.api_version, self.api_version_1)
self.assertEqual(blob_client._client._config.version, self.api_version_1)
async def append_blob(
container: ContainerClient,
blob_name: str,
content_type: str,
content_encoding: str,
data: Any,
return_sas_token: bool = True,
metadata: Dict[str, str] = None,
) -> str:
"""
Uploads the given data to a blob record.
If a blob with the given name already exist, it throws an error.
Returns a uri with a SAS token to access the newly created blob.
"""
await create_container_using_client(container)
logger.info(f"Appending data to blob '{blob_name}'" +
f"in container '{container.container_name}'" +
f"on account: '{container.account_name}'")
content_settings = ContentSettings(content_type=content_type,
content_encoding=content_encoding)
blob = container.get_blob_client(blob_name)
try:
props = await blob.get_blob_properties()
if props.blob_type != BlobType.AppendBlob:
raise Exception("blob must be an append blob")
except exceptions.ResourceNotFoundError:
props = await blob.create_append_blob(
content_settings=content_settings, metadata=metadata)
await blob.append_block(data, len(data))
logger.debug(f" - blob '{blob_name}' appended. generating sas token.")
if return_sas_token:
uri = get_blob_uri_with_sas_token(blob)
else:
uri = remove_sas_token(blob.url)
logger.debug(f" - blob access url: '{uri}'.")
return uri
async def save_to_storage(
data: List[str],
container_client: ContainerClient,
store_param: str,
append=False,
file_append_name: Optional[str] = None,
):
"""
Saves message to storage.
:param data: List of data be stored.
:param container_client: Blob service client (initialized outside this function).
:param store_param: The main folder in the container to store the file in.
:param append: Flag to determine whether the data should be appended to an append blob.
:param file_append_name: Name of the append blob to store to. Ignored if append is False.
:return: Name of the blob stored to and result
"""
# Success flag.
result = False
# Get the blob file name and the data (string) to store.
if not append:
file_name = generate_file_name(store_param=store_param)
else:
file_name = None
# If the file_name is None, we should be using the append blob name.
# If the append blob name is not given, an exception is raised.
if not file_name:
if not file_append_name:
raise StorageTypeConfigurationException(
"SET BLOB TO 'APPEND', YET NO FILE GIVEN FOR THE APPEND BLOB!")
file_name = file_append_name
# Store the data utilizing a simple retry policy.
# In truth, the Blob Client we're using already uses an ExponentialRetry mechanic. This is
# merely an additional fail-safe to it, in case the library at some point changes some of
# the default retry parameters or the save load per second is far bigger than expected.
# In addition, on the off-chance the user is using the TIMED_APPEND option, this retry loop helps with
# potential concurrency issues. If the function manages to get to this point without an append blob
# existing, this loop will give enough time for it to be created in the meantime and ensure a successful
# store.
# In practice, this loop will not execute more than once.
for i in range(STORE_RETRY_POLICY_MAX):
try:
# We include getting the blob client in the retry, due to the fact we likely
# need to renew the lease for the blob.
blob_client = container_client.get_blob_client(file_name, )
if append:
store_method = blob_client.append_block
else:
store_method = blob_client.upload_blob
async with blob_client:
await store_method("\n".join(data))
# Set the result to true.
result = True
# Escape the retry loop.
break
# Currently set to catch a general exception, seeing as how the documentation doesn't
# actually state the possible exceptions that could occur during these processes.
except Exception as exc:
if i == STORE_RETRY_POLICY_MAX - 1:
logging.error(
"FAILED TO SAVE TO STORAGE! | PATH: %s | FAILED MESSAGE CONTENTS: %s | EXCEPTION %s",
file_name,
data,
str(exc),
)
else:
logging.warning(
"FAILED TO SAVE TO STORAGE! | PATH: %s | RETRYING... (ATTEMPT NO. %s)",
file_name,
str(i + 1),
)
await asyncio.sleep(STORE_RETRY_BACKOFF_TIME)
return file_name, result