def __init__(self, settings: BlobStorageSettings):
if not settings.container_name:
raise Exception("Container name is required.")
if settings.connection_string:
blob_service_client = BlobServiceClient.from_connection_string(
settings.connection_string)
else:
blob_service_client = BlobServiceClient.from_connection_string(
convert_account_name_and_key_to_connection_string(settings))
self.__container_client = blob_service_client.get_container_client(
settings.container_name)
self.__initialized = False
async def _test_blob_service_properties_async(self):
# Instantiate a BlobServiceClient using a connection string
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(self.connection_string)
# [START set_blob_service_properties]
# Create service properties
from azure.storage.blob import BlobAnalyticsLogging, Metrics, CorsRule, RetentionPolicy
# Create logging settings
logging = BlobAnalyticsLogging(read=True, write=True, delete=True, retention_policy=RetentionPolicy(enabled=True, days=5))
# Create metrics for requests statistics
hour_metrics = Metrics(enabled=True, include_apis=True, retention_policy=RetentionPolicy(enabled=True, days=5))
minute_metrics = Metrics(enabled=True, include_apis=True,
retention_policy=RetentionPolicy(enabled=True, days=5))
# Create CORS rules
cors_rule = CorsRule(['www.xyz.com'], ['GET'])
cors = [cors_rule]
# Set the service properties
await blob_service_client.set_service_properties(logging, hour_metrics, minute_metrics, cors)
# [END set_blob_service_properties]
# [START get_blob_service_properties]
properties = await blob_service_client.get_service_properties()
# [END get_blob_service_properties]
assert properties is not None
async def write_to_blob(stream, name, metadata={}):
blob_service_client = BlobServiceClient.from_connection_string(
os.environ["BLOB_CONN_STR"])
async with blob_service_client:
# Instantiate a new ContainerClient
container_client = blob_service_client.get_container_client(
os.environ["CONTAINER_NAME"])
# Instantiate a new BlobClient
blob_client = container_client.get_blob_client(name)
# [START upload_a_blob in CHUNKSIZE chunks ]
blocks = []
chunk_num = 0
async for data in chunk_stream(stream):
chunk_id = _id(chunk_num, name)
logging.info(
f"Chunk `{chunk_id}` len {len(data)} to blob name: {name}")
await blob_client.stage_block(block_id=chunk_id,
data=data,
length=len(data))
blocks.append(chunk_id)
chunk_num += 1
blob = await blob_client.commit_block_list(block_list=blocks,
metadata=metadata)
# [END upload_a_blob]
logging.info(f'Blob {name} uploaded in {chunk_num} parts')
async def _test_container_sample_async(self):
# [START create_container_client_from_service]
# Instantiate a BlobServiceClient using a connection string
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(
self.connection_string)
# Instantiate a ContainerClient
container_client = blob_service_client.get_container_client(
"mynewcontainerasync")
# [END create_container_client_from_service]
# [START create_container_client_sasurl]
from azure.storage.blob.aio import ContainerClient
sas_url = sas_url = "https://account.blob.core.windows.net/mycontainer?sv=2015-04-05&st=2015-04-29T22%3A18%3A26Z&se=2015-04-30T02%3A23%3A26Z&sr=b&sp=rw&sip=168.1.5.60-168.1.5.70&spr=https&sig=Z%2FRHIX5Xcg0Mq2rqI3OlWTjEg2tYkboXr1P9ZUXDtkk%3D"
container = ContainerClient.from_container_url(sas_url)
# [END create_container_client_sasurl]
try:
# [START create_container]
await container_client.create_container()
# [END create_container]
# [START get_container_properties]
properties = await container_client.get_container_properties()
# [END get_container_properties]
assert properties is not None
finally:
# [START delete_container]
await container_client.delete_container()
async def delete_multiple_blobs_async(self):
# Instantiate a BlobServiceClient using a connection string
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(self.connection_string)
async with blob_service_client:
# Instantiate a ContainerClient
container_client = blob_service_client.get_container_client("containerforbatchblobdeletesasync")
# Create new Container
try:
await container_client.create_container()
except ResourceExistsError:
# Container already created
pass
# Upload a blob to the container
upload_data = b"Hello World"
await container_client.upload_blob(name="my_blob1", data=upload_data)
await container_client.upload_blob(name="my_blob2", data=upload_data)
await container_client.upload_blob(name="my_blob3", data=upload_data)
# [START delete_multiple_blobs]
# Delete multiple blobs in the container by name
await container_client.delete_blobs("my_blob1", "my_blob2")
# Delete multiple blobs by properties iterator
my_blobs = container_client.list_blobs(name_starts_with="my_blob")
await container_client.delete_blobs(*[b async for b in my_blobs]) # async for in list comprehension after 3.6 only
# [END delete_multiple_blobs]
# Delete container
await blob_service_client.delete_container("containerforbatchblobdeletesasync")
async def list_blobs(self):
async with AsyncBlobServiceClient.from_connection_string(
self._connection_string) as client:
container: AsyncContainerClient = client.get_container_client(
self.container)
async for blob in container.list_blobs(name_starts_with=self.path):
yield blob
async def page_blob_sample_async(self):
# Instantiate a new BlobServiceClient using a connection string
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(self.connection_string)
async with blob_service_client:
# Instantiate a new ContainerClient
container_client = blob_service_client.get_container_client("mypagecontainerasync")
try:
# Create new Container in the Service
await container_client.create_container()
# Instantiate a new BlobClient
blob_client = container_client.get_blob_client("mypageblob")
# Upload content to the Page Blob
data = b'abcd'*128
await blob_client.upload_blob(data, blob_type="PageBlob")
# Download Page Blob
with open(DEST_FILE, "wb") as my_blob:
stream = await blob_client.download_blob()
data = await stream.readall()
my_blob.write(data)
# Delete Page Blob
await blob_client.delete_blob()
finally:
# Delete container
await container_client.delete_container()
def __init__(
self,
config: Dict,
):
# If output binding or storage queue mode is enabled, there is no point in even instantiating a blob client on
# the endpoint.
self.output_binding = config.get("OUTPUT_BINDING", False)
if not self.output_binding:
blob_service_client = BlobServiceClient.from_connection_string(
conn_str=config["STORAGE_CONNECTION_STRING"])
self.container_client = blob_service_client.get_container_client(
config["CONTAINER_NAME"])
else:
self.container_client = None
self.get_metadata = config.get("GET_METADATA", False)
self.append = config.get("APPEND", False)
self.timed_append = config.get("TIMED_APPEND", False)
# Just in case the user forgot to configure the APPEND config variable.
if self.timed_append:
self.append = True
self.file_name = None
if self.append:
self.append_lock = asyncio.Lock()
self.now = None
async def _test_append_blob_sample_async(self):
# Instantiate a new BlobServiceClient using a connection string
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(
self.connection_string)
# Instantiate a new ContainerClient
container_client = blob_service_client.get_container_client(
"myappendcontainerasync")
try:
# Create new Container in the Service
await container_client.create_container()
# Get the BlobClient
blob_client = container_client.get_blob_client("myappendblob")
# Upload content to the append blob
with open(SOURCE_FILE, "rb") as data:
await blob_client.upload_blob(data, blob_type="AppendBlob")
# Download append blob
with open(DEST_FILE, "wb") as my_blob:
stream = await blob_client.download_blob()
data = await stream.content_as_bytes()
my_blob.write(data)
# Delete append blob
await blob_client.delete_blob()
finally:
# Delete container
await container_client.delete_container()
async def _test_block_blob_sample_async(self):
# Instantiate a new BlobServiceClient using a connection string
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(self.connection_string)
# Instantiate a new ContainerClient
container_client = blob_service_client.get_container_client("myblockcontainerasync")
try:
# Create new Container in the service
await container_client.create_container()
# Instantiate a new BlobClient
blob_client = container_client.get_blob_client("myblockblob")
# [START upload_a_blob]
# Upload content to block blob
with open(SOURCE_FILE, "rb") as data:
await blob_client.upload_blob(data, blob_type="BlockBlob")
# [END upload_a_blob]
# [START download_a_blob]
with open(DEST_FILE, "wb") as my_blob:
stream = await blob_client.download_blob()
data = await stream.readall()
my_blob.write(data)
# [END download_a_blob]
# [START delete_blob]
await blob_client.delete_blob()
# [END delete_blob]
finally:
# Delete the container
await container_client.delete_container()
async def _test_blob_snapshots_async(self):
# Instantiate a BlobServiceClient using a connection string
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(
self.connection_string)
# Instantiate a ContainerClient
container_client = blob_service_client.get_container_client(
"containerformyblobsasync")
# Create new Container
await container_client.create_container()
# Upload a blob to the container
with open(SOURCE_FILE, "rb") as data:
await container_client.upload_blob(name="my_blob", data=data)
# Get a BlobClient for a specific blob
blob_client = blob_service_client.get_blob_client(
container="containerformyblobsasync", blob="my_blob")
# [START create_blob_snapshot]
# Create a read-only snapshot of the blob at this point in time
snapshot_blob = await blob_client.create_snapshot()
# Get the snapshot ID
print(snapshot_blob.get('snapshot'))
# Delete only the snapshot (blob itself is retained)
await blob_client.delete_blob(delete_snapshots="only")
# [END create_blob_snapshot]
# Delete container
await blob_service_client.delete_container("containerformyblobsasync")
async def main(forminfo) -> str:
result = forminfo.get("result")
path = forminfo.get("path")
# Get blob name and container from path
container = path.split("/")[0]
blob = "/".join(path.split("/")[1:])
# Download blob from blob storage
blob_service_client = BlobServiceClient.from_connection_string(
os.environ["StorageAccount"])
async with blob_service_client:
filename = blob + ".json"
blob_container_client = blob_service_client.get_container_client(
"output")
await blob_container_client.upload_blob(
filename,
json.dumps(result),
content_settings=ContentSettings(content_type="application/json"),
)
return f"output/{blob}!"
async def main():
try:
CONNECTION_STRING = os.environ['AZURE_STORAGE_CONNECTION_STRING']
except KeyError:
print("AZURE_STORAGE_CONNECTION_STRING must be set.")
sys.exit(1)
status = None
blob_service_client = BlobServiceClient.from_connection_string(
CONNECTION_STRING)
source_blob = "http://www.gutenberg.org/files/59466/59466-0.txt"
copied_blob = blob_service_client.get_blob_client("mycontainer",
'59466-0.txt')
# Copy started"
await copied_blob.start_copy_from_url(source_blob)
for i in range(10):
props = await copied_blob.get_blob_properties()
status = props.copy.status
print("Copy status: " + status)
if status == "success":
# copy finished
break
time.sleep(10)
if status != "success":
# if not finished after 100s, cancel the operation
props = await copied_blob.get_blob_properties()
print(props.copy.status)
copy_id = props.copy.id
await copied_blob.abort_copy(copy_id)
props = copied_blob.get_blob_properties()
print(props.copy.status)
async def _test_acquire_lease_on_blob_async(self):
# Instantiate a BlobServiceClient using a connection string
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(
self.connection_string)
# Instantiate a ContainerClient
container_client = blob_service_client.get_container_client(
"leasemyblobscontainerasync")
# Create new Container
await container_client.create_container()
# Upload a blob to the container
with open(SOURCE_FILE, "rb") as data:
await container_client.upload_blob(name="my_blob", data=data)
# [START acquire_lease_on_blob]
# Get the blob client
blob_client = blob_service_client.get_blob_client(
"leasemyblobscontainerasync", "my_blob")
# Acquire a lease on the blob
lease = await blob_client.acquire_lease()
# Delete blob by passing in the lease
await blob_client.delete_blob(lease=lease)
# [END acquire_lease_on_blob]
# Delete container
await blob_service_client.delete_container("leasemyblobscontainerasync"
)
async def _test_set_metadata_on_container_async(self):
# Instantiate a BlobServiceClient using a connection string
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(
self.connection_string)
# Instantiate a ContainerClient
container_client = blob_service_client.get_container_client(
"mymetadatacontainerasync")
try:
# Create new Container
await container_client.create_container()
# [START set_container_metadata]
# Create key, value pairs for metadata
metadata = {'type': 'test'}
# Set metadata on the container
await container_client.set_container_metadata(metadata=metadata)
# [END set_container_metadata]
# Get container properties
properties = (await
container_client.get_container_properties()).metadata
assert properties == metadata
finally:
# Delete container
await container_client.delete_container()
async def process_blob_files(self, src_container, dest_container,
local_temp_dir):
try:
self.init(local_temp_dir)
# Create the BlobServiceClient object which will be used to create a container client
blob_service_client = BlobServiceClient.from_connection_string(
self.conn_str)
async with blob_service_client:
container_client = blob_service_client.get_container_client(
src_container)
tasks = []
async for blob in container_client.list_blobs():
blob_client = container_client.get_blob_client(blob.name)
local_file = "{}/{}".format(local_temp_dir,
os.path.basename(blob.name))
tasks.append(
self.process_blob_file(blob_client, dest_container,
local_file))
await asyncio.gather(*tasks)
self.cleanup(local_temp_dir)
except Exception as e:
print(repr(e))
sys.exit(1)
def __init__(self, app):
self._app = app
if "azureBlobClient" in app:
if "token_expiration" in app:
# TBD - does this apply for Azure?
# check that our token is not about to expire
expiration = app["token_expiration"]
now = datetime.datetime.now()
delta = expiration - now
if delta.total_seconds() > 10:
self._client = app["azureBlobClient"]
return
# otherwise, fall through and get a new token
log.info("Azure access token has expired - renewing")
else:
self._client = app["azureBlobClient"]
return
# first time setup of Azure client or limited time token has expired
# TBD - what do do about region?
log.info("AzureBlobClient init")
azure_connection_string = config.get('azure_connection_string')
if not azure_connection_string:
msg="No connection string specified"
log.error(msg)
raise ValueError(msg)
log.info(f"Using azure_connection_string: {azure_connection_string}")
self._client = BlobServiceClient.from_connection_string(azure_connection_string)
app['azureBlobClient'] = self._client # save so same client can be returned in subsequent calls
async def _test_list_blobs_in_container_async(self):
# Instantiate a BlobServiceClient using a connection string
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(
self.connection_string)
# Instantiate a ContainerClient
container_client = blob_service_client.get_container_client(
"myblobscontainerasync")
# Create new Container
await container_client.create_container()
# [START upload_blob_to_container]
with open(SOURCE_FILE, "rb") as data:
blob_client = await container_client.upload_blob(name="blobby",
data=data)
properties = await blob_client.get_blob_properties()
# [END upload_blob_to_container]
# [START list_blobs_in_container]
blobs_list = []
async for blob in container_client.list_blobs():
blobs_list.append(blob)
# [END list_blobs_in_container]
assert blobs_list is not None
# Delete container
await container_client.delete_container()
def from_connection_string(
cls, connection_string: str,
container_name: str) -> "AzureBlobAsyncFileStore":
return cls(
BlobServiceClient.from_connection_string(
connection_string), # type: ignore
container_name,
)
async def reset():
storage = BlobServiceClient.from_connection_string(
BLOB_STORAGE_SETTINGS.connection_string
)
try:
await storage.delete_container(BLOB_STORAGE_SETTINGS.container_name)
except ResourceNotFoundError:
pass
async def container_access_policy_async(self):
# Instantiate a BlobServiceClient using a connection string
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(
self.connection_string)
async with blob_service_client:
# Instantiate a ContainerClient
container_client = blob_service_client.get_container_client(
"myaccesscontainerasync")
try:
# Create new Container
await container_client.create_container()
# [START set_container_access_policy]
# Create access policy
from azure.storage.blob import AccessPolicy, ContainerSasPermissions
access_policy = AccessPolicy(
permission=ContainerSasPermissions(read=True),
expiry=datetime.utcnow() + timedelta(hours=1),
start=datetime.utcnow() - timedelta(minutes=1))
identifiers = {'test': access_policy}
# Set the access policy on the container
await container_client.set_container_access_policy(
signed_identifiers=identifiers)
# [END set_container_access_policy]
# [START get_container_access_policy]
policy = await container_client.get_container_access_policy()
# [END get_container_access_policy]
# [START generate_sas_token]
# Use access policy to generate a sas token
from azure.storage.blob import generate_container_sas
sas_token = generate_container_sas(
container_client.account_name,
container_client.container_name,
account_key=container_client.credential.account_key,
policy_id='my-access-policy-id')
# [END generate_sas_token]
# Use the sas token to authenticate a new client
# [START create_container_client_sastoken]
from azure.storage.blob.aio import ContainerClient
container = ContainerClient.from_container_url(
container_url=
"https://account.blob.core.windows.net/mycontainerasync",
credential=sas_token,
)
# [END create_container_client_sastoken]
finally:
# Delete container
await container_client.delete_container()
async def blob_service_stats_async(self):
# Instantiate a BlobServiceClient using a connection string
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(
self.connection_string)
# [START get_blob_service_stats]
stats = await blob_service_client.get_service_stats()
async def _test_container_access_policy_async(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Instantiate a BlobServiceClient using a connection string
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(
self.connection_string)
# Instantiate a ContainerClient
container_client = blob_service_client.get_container_client(
"myaccesscontainerasync")
try:
# Create new Container
await container_client.create_container()
# [START set_container_access_policy]
# Create access policy
from azure.storage.blob import AccessPolicy, ContainerSasPermissions
access_policy = AccessPolicy(
permission=ContainerSasPermissions(read=True),
expiry=datetime.utcnow() + timedelta(hours=1),
start=datetime.utcnow() - timedelta(minutes=1))
identifiers = {'test': access_policy}
# Set the access policy on the container
await container_client.set_container_access_policy(
signed_identifiers=identifiers)
# [END set_container_access_policy]
# [START get_container_access_policy]
policy = await container_client.get_container_access_policy()
# [END get_container_access_policy]
# [START generate_sas_token]
# Use access policy to generate a sas token
sas_token = container_client.generate_shared_access_signature(
policy_id='my-access-policy-id')
# [END generate_sas_token]
# Use the sas token to authenticate a new client
# [START create_container_client_sastoken]
from azure.storage.blob.aio import ContainerClient
container = ContainerClient.from_container_url(
container_url=
"https://account.blob.core.windows.net/mycontainerasync",
credential=sas_token,
)
# [END create_container_client_sastoken]
finally:
# Delete container
await container_client.delete_container()
async def get_storage_account_information_async(self):
# Instantiate a BlobServiceClient using a connection string
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(
self.connection_string)
# [START get_blob_service_account_info]
account_info = await blob_service_client.get_account_information()
print('Using Storage SKU: {}'.format(account_info['sku_name']))
async def main():
try:
async with BlobServiceClient.from_connection_string(
CONNECTION_STRING) as service_client:
containers = service_client.list_containers()
async for container in containers:
await walk_container(service_client, container)
except Exception as error:
print(error)
sys.exit(1)
def __init__(self, arguments):
super().__init__(arguments)
connection_string = self.get_from_env("AZURE_STORAGE_CONNECTION_STRING")
kwargs = {}
kwargs['max_single_put_size'] = self.args.max_put_size
kwargs['max_block_size'] = self.args.max_block_size
kwargs['min_large_block_upload_threshold'] = self.args.buffer_threshold
if not _ServiceTest.service_client or self.args.no_client_share:
_ServiceTest.service_client = SyncBlobServiceClient.from_connection_string(conn_str=connection_string, **kwargs)
_ServiceTest.async_service_client = AsyncBlobServiceClient.from_connection_string(conn_str=connection_string, **kwargs)
self.service_client = _ServiceTest.service_client
self.async_service_client =_ServiceTest.async_service_client
async def upload_decompressed_files(self, dest_container,
decompression_dir, metadata_json):
blob_service_client = BlobServiceClient.from_connection_string(
self.conn_str)
files_to_upload = self.get_files_to_upload(decompression_dir)
async with blob_service_client:
for file in files_to_upload:
await self.upload_decompressed_file(blob_service_client, file,
dest_container,
metadata_json)
async def create_container(connection_string: str,
container_name: str) -> ContainerClient:
"""
Creates and initialize a container; returns the client needed to access it.
"""
blob_service_client = BlobServiceClient.from_connection_string(
connection_string)
logger.info(f'{"Initializing storage client for account:"}' +
f"{blob_service_client.account_name}")
container_client = blob_service_client.get_container_client(container_name)
await create_container_using_client(container_client)
return container_client
async def stream_block_blob(self):
import uuid
# Instantiate a new BlobServiceClient using a connection string - set chunk size to 1MB
from azure.storage.blob import BlobBlock
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(
self.connection_string,
max_single_get_size=1024 * 1024,
max_chunk_get_size=1024 * 1024)
async with blob_service_client:
# Instantiate a new ContainerClient
container_client = blob_service_client.get_container_client(
"containerasync")
# Generate 4MB of data
data = b'a' * 4 * 1024 * 1024
try:
# Create new Container in the service
await container_client.create_container()
# Instantiate a new source blob client
source_blob_client = container_client.get_blob_client(
"source_blob")
# Upload content to block blob
await source_blob_client.upload_blob(data,
blob_type="BlockBlob")
destination_blob_client = container_client.get_blob_client(
"destination_blob")
# This returns a StorageStreamDownloader.
stream = await source_blob_client.download_blob()
block_list = []
# Read data in chunks to avoid loading all into memory at once
async for chunk in stream.chunks():
# process your data (anything can be done here really. `chunk` is a byte array).
block_id = str(uuid.uuid4())
await destination_blob_client.stage_block(
block_id=block_id, data=chunk)
block_list.append(BlobBlock(block_id=block_id))
# Upload the whole chunk to azure storage and make up one blob
await destination_blob_client.commit_block_list(block_list)
finally:
# Delete container
await container_client.delete_container()
def __init__(self, blobonedge_module_name, blob_account_name,
blob_account_key, sound_container_name, edge_id):
localblob_connectionstring = 'DefaultEndpointsProtocol=http;BlobEndpoint=http://' + blobonedge_module_name + ':11002/' + blob_account_name + ';AccountName=' + blob_account_name + ';AccountKey=' + blob_account_key + ';'
print("Try to connect to blob on edge by " +
localblob_connectionstring)
self.blobServiceClient = BlobServiceClient.from_connection_string(
localblob_connectionstring)
print("Connected to blob on edge")
self.containerClient = self.blobServiceClient.get_container_client(
sound_container_name)
self.soundContainerName = sound_container_name
self.edgeId = edge_id