async def upload_blob( container: ContainerClient, blob_name: str, content_type: str, content_encoding: str, data: Any, return_sas_token: bool = True, ) -> str: """ Uploads the given data to a blob record. If a blob with the given name already exist, it throws an error. Returns a uri with a SAS token to access the newly created blob. """ await create_container_using_client(container) logger.info(f"Uploading blob '{blob_name}'" + f"to container '{container.container_name}'" + f"on account: '{container.account_name}'") content_settings = ContentSettings(content_type=content_type, content_encoding=content_encoding) blob = container.get_blob_client(blob_name) await blob.upload_blob(data, content_settings=content_settings) logger.debug(f" - blob '{blob_name}' uploaded. generating sas token.") if return_sas_token: uri = get_blob_uri_with_sas_token(blob) else: uri = remove_sas_token(blob.url) logger.debug(f" - blob access url: '{uri}'.") return uri
async def generate_append_blob( container_client: ContainerClient, manager: ServiceManagerBase, store_param: str, time_based=False, ): """ Generates an append blob, either based on a random UUID or the current time (hour and minute). :param container_client: Container Client. :param manager: The manager calling for the new file name generation. :param store_param: Name of the folder the persistor will store the data to. :param time_based: Whether or not the blob's name should be generated based on the current time. :return: Complete file path to store the blob to. """ now = datetime.now() # If time-based, set the file name based on time (hour/minute). if time_based: name = "{hour}-{minute}".format(hour=str(now.hour), minute=str(now.minute)) # Otherwise, generate the name based on a random UUID else: name = str(uuid.uuid4()) file_name = generate_file_name(store_param, name) # To help minimize concurrency issues, we introduce an async lock within the same instance. # (Not much we can do with concurrent instances/workers.) async with manager.append_lock: if not time_based or not manager.now or now.hour > manager.now.hour or now.minute > manager.now.minute: blob_client = container_client.get_blob_client(file_name) # Try to get the blob properties. If the blob doesn't exist, this will return an exception. try: await blob_client.get_blob_properties() except ResourceNotFoundError: # Since the blob doesn't appear to exist, try to create a new one. # Since we need to think about concurrency between different instances that # don't share the manager object, # we have to assume some exception will occur when multiple creation attempts are made. try: await blob_client.create_append_blob() # Unsure which exception might be thrown if concurrently attempting to create the append blob. except: pass manager.now = now manager.file_name = file_name return file_name
def __init__( self, container: ContainerClient, blob_name: str, content_type: str, content_encoding: str, ): self.container = container self.blob_name = blob_name self.content_settings = ContentSettings( content_type=content_type, content_encoding=content_encoding) self.state = StreamedBlobState.not_initialized self.blob = container.get_blob_client(blob_name) self.blocks = []
async def init_blob_for_streaming_upload( container: ContainerClient, blob_name: str, content_type: str, content_encoding: str, data: Any, return_sas_token: bool = True, ) -> str: """ Uploads the given data to a blob record. If a blob with the given name already exist, it throws an error. Returns a uri with a SAS token to access the newly created blob. """ await create_container_using_client(container) logger.info(f"Streaming blob '{blob_name}'" + f"to container '{container.container_name}' on account:" + f"'{container.account_name}'") content_settings = ContentSettings(content_type=content_type, content_encoding=content_encoding) blob = container.get_blob_client(blob_name) await blob.stage_block() await blob.commit_block_list() await blob.upload_blob(data, content_settings=content_settings) logger.debug(f" - blob '{blob_name}' uploaded. generating sas token.") if return_sas_token: sas_token = generate_blob_sas( blob.account_name, blob.container_name, blob.blob_name, account_key=blob.credential.account_key, permission=BlobSasPermissions(read=True), expiry=datetime.utcnow() + timedelta(days=14), ) uri = blob.url + "?" + sas_token else: uri = remove_sas_token(blob.url) logger.debug(f" - blob access url: '{uri}'.") return uri
def test_container_client_api_version_property(self): container_client = ContainerClient( "https://foo.blob.core.windows.net/account", self.container_name, credential="fake_key") self.assertEqual(container_client.api_version, self.api_version_2) self.assertEqual(container_client._client._config.version, self.api_version_2) container_client = ContainerClient( "https://foo.blob.core.windows.net/account", self.container_name, credential="fake_key", api_version=self.api_version_1) self.assertEqual(container_client.api_version, self.api_version_1) self.assertEqual(container_client._client._config.version, self.api_version_1) blob_client = container_client.get_blob_client("foo") self.assertEqual(blob_client.api_version, self.api_version_1) self.assertEqual(blob_client._client._config.version, self.api_version_1)
async def append_blob( container: ContainerClient, blob_name: str, content_type: str, content_encoding: str, data: Any, return_sas_token: bool = True, metadata: Dict[str, str] = None, ) -> str: """ Uploads the given data to a blob record. If a blob with the given name already exist, it throws an error. Returns a uri with a SAS token to access the newly created blob. """ await create_container_using_client(container) logger.info(f"Appending data to blob '{blob_name}'" + f"in container '{container.container_name}'" + f"on account: '{container.account_name}'") content_settings = ContentSettings(content_type=content_type, content_encoding=content_encoding) blob = container.get_blob_client(blob_name) try: props = await blob.get_blob_properties() if props.blob_type != BlobType.AppendBlob: raise Exception("blob must be an append blob") except exceptions.ResourceNotFoundError: props = await blob.create_append_blob( content_settings=content_settings, metadata=metadata) await blob.append_block(data, len(data)) logger.debug(f" - blob '{blob_name}' appended. generating sas token.") if return_sas_token: uri = get_blob_uri_with_sas_token(blob) else: uri = remove_sas_token(blob.url) logger.debug(f" - blob access url: '{uri}'.") return uri
async def save_to_storage( data: List[str], container_client: ContainerClient, store_param: str, append=False, file_append_name: Optional[str] = None, ): """ Saves message to storage. :param data: List of data be stored. :param container_client: Blob service client (initialized outside this function). :param store_param: The main folder in the container to store the file in. :param append: Flag to determine whether the data should be appended to an append blob. :param file_append_name: Name of the append blob to store to. Ignored if append is False. :return: Name of the blob stored to and result """ # Success flag. result = False # Get the blob file name and the data (string) to store. if not append: file_name = generate_file_name(store_param=store_param) else: file_name = None # If the file_name is None, we should be using the append blob name. # If the append blob name is not given, an exception is raised. if not file_name: if not file_append_name: raise StorageTypeConfigurationException( "SET BLOB TO 'APPEND', YET NO FILE GIVEN FOR THE APPEND BLOB!") file_name = file_append_name # Store the data utilizing a simple retry policy. # In truth, the Blob Client we're using already uses an ExponentialRetry mechanic. This is # merely an additional fail-safe to it, in case the library at some point changes some of # the default retry parameters or the save load per second is far bigger than expected. # In addition, on the off-chance the user is using the TIMED_APPEND option, this retry loop helps with # potential concurrency issues. If the function manages to get to this point without an append blob # existing, this loop will give enough time for it to be created in the meantime and ensure a successful # store. # In practice, this loop will not execute more than once. for i in range(STORE_RETRY_POLICY_MAX): try: # We include getting the blob client in the retry, due to the fact we likely # need to renew the lease for the blob. blob_client = container_client.get_blob_client(file_name, ) if append: store_method = blob_client.append_block else: store_method = blob_client.upload_blob async with blob_client: await store_method("\n".join(data)) # Set the result to true. result = True # Escape the retry loop. break # Currently set to catch a general exception, seeing as how the documentation doesn't # actually state the possible exceptions that could occur during these processes. except Exception as exc: if i == STORE_RETRY_POLICY_MAX - 1: logging.error( "FAILED TO SAVE TO STORAGE! | PATH: %s | FAILED MESSAGE CONTENTS: %s | EXCEPTION %s", file_name, data, str(exc), ) else: logging.warning( "FAILED TO SAVE TO STORAGE! | PATH: %s | RETRYING... (ATTEMPT NO. %s)", file_name, str(i + 1), ) await asyncio.sleep(STORE_RETRY_BACKOFF_TIME) return file_name, result