def __init__(self, settings: BlobStorageSettings): if not settings.container_name: raise Exception("Container name is required.") if settings.connection_string: blob_service_client = BlobServiceClient.from_connection_string( settings.connection_string) else: blob_service_client = BlobServiceClient.from_connection_string( convert_account_name_and_key_to_connection_string(settings)) self.__container_client = blob_service_client.get_container_client( settings.container_name) self.__initialized = False
async def _test_blob_service_properties_async(self): # Instantiate a BlobServiceClient using a connection string from azure.storage.blob.aio import BlobServiceClient blob_service_client = BlobServiceClient.from_connection_string(self.connection_string) # [START set_blob_service_properties] # Create service properties from azure.storage.blob import BlobAnalyticsLogging, Metrics, CorsRule, RetentionPolicy # Create logging settings logging = BlobAnalyticsLogging(read=True, write=True, delete=True, retention_policy=RetentionPolicy(enabled=True, days=5)) # Create metrics for requests statistics hour_metrics = Metrics(enabled=True, include_apis=True, retention_policy=RetentionPolicy(enabled=True, days=5)) minute_metrics = Metrics(enabled=True, include_apis=True, retention_policy=RetentionPolicy(enabled=True, days=5)) # Create CORS rules cors_rule = CorsRule(['www.xyz.com'], ['GET']) cors = [cors_rule] # Set the service properties await blob_service_client.set_service_properties(logging, hour_metrics, minute_metrics, cors) # [END set_blob_service_properties] # [START get_blob_service_properties] properties = await blob_service_client.get_service_properties() # [END get_blob_service_properties] assert properties is not None
async def write_to_blob(stream, name, metadata={}): blob_service_client = BlobServiceClient.from_connection_string( os.environ["BLOB_CONN_STR"]) async with blob_service_client: # Instantiate a new ContainerClient container_client = blob_service_client.get_container_client( os.environ["CONTAINER_NAME"]) # Instantiate a new BlobClient blob_client = container_client.get_blob_client(name) # [START upload_a_blob in CHUNKSIZE chunks ] blocks = [] chunk_num = 0 async for data in chunk_stream(stream): chunk_id = _id(chunk_num, name) logging.info( f"Chunk `{chunk_id}` len {len(data)} to blob name: {name}") await blob_client.stage_block(block_id=chunk_id, data=data, length=len(data)) blocks.append(chunk_id) chunk_num += 1 blob = await blob_client.commit_block_list(block_list=blocks, metadata=metadata) # [END upload_a_blob] logging.info(f'Blob {name} uploaded in {chunk_num} parts')
async def _test_container_sample_async(self): # [START create_container_client_from_service] # Instantiate a BlobServiceClient using a connection string from azure.storage.blob.aio import BlobServiceClient blob_service_client = BlobServiceClient.from_connection_string( self.connection_string) # Instantiate a ContainerClient container_client = blob_service_client.get_container_client( "mynewcontainerasync") # [END create_container_client_from_service] # [START create_container_client_sasurl] from azure.storage.blob.aio import ContainerClient sas_url = sas_url = "https://account.blob.core.windows.net/mycontainer?sv=2015-04-05&st=2015-04-29T22%3A18%3A26Z&se=2015-04-30T02%3A23%3A26Z&sr=b&sp=rw&sip=168.1.5.60-168.1.5.70&spr=https&sig=Z%2FRHIX5Xcg0Mq2rqI3OlWTjEg2tYkboXr1P9ZUXDtkk%3D" container = ContainerClient.from_container_url(sas_url) # [END create_container_client_sasurl] try: # [START create_container] await container_client.create_container() # [END create_container] # [START get_container_properties] properties = await container_client.get_container_properties() # [END get_container_properties] assert properties is not None finally: # [START delete_container] await container_client.delete_container()
async def delete_multiple_blobs_async(self): # Instantiate a BlobServiceClient using a connection string from azure.storage.blob.aio import BlobServiceClient blob_service_client = BlobServiceClient.from_connection_string(self.connection_string) async with blob_service_client: # Instantiate a ContainerClient container_client = blob_service_client.get_container_client("containerforbatchblobdeletesasync") # Create new Container try: await container_client.create_container() except ResourceExistsError: # Container already created pass # Upload a blob to the container upload_data = b"Hello World" await container_client.upload_blob(name="my_blob1", data=upload_data) await container_client.upload_blob(name="my_blob2", data=upload_data) await container_client.upload_blob(name="my_blob3", data=upload_data) # [START delete_multiple_blobs] # Delete multiple blobs in the container by name await container_client.delete_blobs("my_blob1", "my_blob2") # Delete multiple blobs by properties iterator my_blobs = container_client.list_blobs(name_starts_with="my_blob") await container_client.delete_blobs(*[b async for b in my_blobs]) # async for in list comprehension after 3.6 only # [END delete_multiple_blobs] # Delete container await blob_service_client.delete_container("containerforbatchblobdeletesasync")
async def list_blobs(self): async with AsyncBlobServiceClient.from_connection_string( self._connection_string) as client: container: AsyncContainerClient = client.get_container_client( self.container) async for blob in container.list_blobs(name_starts_with=self.path): yield blob
async def page_blob_sample_async(self): # Instantiate a new BlobServiceClient using a connection string from azure.storage.blob.aio import BlobServiceClient blob_service_client = BlobServiceClient.from_connection_string(self.connection_string) async with blob_service_client: # Instantiate a new ContainerClient container_client = blob_service_client.get_container_client("mypagecontainerasync") try: # Create new Container in the Service await container_client.create_container() # Instantiate a new BlobClient blob_client = container_client.get_blob_client("mypageblob") # Upload content to the Page Blob data = b'abcd'*128 await blob_client.upload_blob(data, blob_type="PageBlob") # Download Page Blob with open(DEST_FILE, "wb") as my_blob: stream = await blob_client.download_blob() data = await stream.readall() my_blob.write(data) # Delete Page Blob await blob_client.delete_blob() finally: # Delete container await container_client.delete_container()
def __init__( self, config: Dict, ): # If output binding or storage queue mode is enabled, there is no point in even instantiating a blob client on # the endpoint. self.output_binding = config.get("OUTPUT_BINDING", False) if not self.output_binding: blob_service_client = BlobServiceClient.from_connection_string( conn_str=config["STORAGE_CONNECTION_STRING"]) self.container_client = blob_service_client.get_container_client( config["CONTAINER_NAME"]) else: self.container_client = None self.get_metadata = config.get("GET_METADATA", False) self.append = config.get("APPEND", False) self.timed_append = config.get("TIMED_APPEND", False) # Just in case the user forgot to configure the APPEND config variable. if self.timed_append: self.append = True self.file_name = None if self.append: self.append_lock = asyncio.Lock() self.now = None
async def _test_append_blob_sample_async(self): # Instantiate a new BlobServiceClient using a connection string from azure.storage.blob.aio import BlobServiceClient blob_service_client = BlobServiceClient.from_connection_string( self.connection_string) # Instantiate a new ContainerClient container_client = blob_service_client.get_container_client( "myappendcontainerasync") try: # Create new Container in the Service await container_client.create_container() # Get the BlobClient blob_client = container_client.get_blob_client("myappendblob") # Upload content to the append blob with open(SOURCE_FILE, "rb") as data: await blob_client.upload_blob(data, blob_type="AppendBlob") # Download append blob with open(DEST_FILE, "wb") as my_blob: stream = await blob_client.download_blob() data = await stream.content_as_bytes() my_blob.write(data) # Delete append blob await blob_client.delete_blob() finally: # Delete container await container_client.delete_container()
async def _test_block_blob_sample_async(self): # Instantiate a new BlobServiceClient using a connection string from azure.storage.blob.aio import BlobServiceClient blob_service_client = BlobServiceClient.from_connection_string(self.connection_string) # Instantiate a new ContainerClient container_client = blob_service_client.get_container_client("myblockcontainerasync") try: # Create new Container in the service await container_client.create_container() # Instantiate a new BlobClient blob_client = container_client.get_blob_client("myblockblob") # [START upload_a_blob] # Upload content to block blob with open(SOURCE_FILE, "rb") as data: await blob_client.upload_blob(data, blob_type="BlockBlob") # [END upload_a_blob] # [START download_a_blob] with open(DEST_FILE, "wb") as my_blob: stream = await blob_client.download_blob() data = await stream.readall() my_blob.write(data) # [END download_a_blob] # [START delete_blob] await blob_client.delete_blob() # [END delete_blob] finally: # Delete the container await container_client.delete_container()
async def _test_blob_snapshots_async(self): # Instantiate a BlobServiceClient using a connection string from azure.storage.blob.aio import BlobServiceClient blob_service_client = BlobServiceClient.from_connection_string( self.connection_string) # Instantiate a ContainerClient container_client = blob_service_client.get_container_client( "containerformyblobsasync") # Create new Container await container_client.create_container() # Upload a blob to the container with open(SOURCE_FILE, "rb") as data: await container_client.upload_blob(name="my_blob", data=data) # Get a BlobClient for a specific blob blob_client = blob_service_client.get_blob_client( container="containerformyblobsasync", blob="my_blob") # [START create_blob_snapshot] # Create a read-only snapshot of the blob at this point in time snapshot_blob = await blob_client.create_snapshot() # Get the snapshot ID print(snapshot_blob.get('snapshot')) # Delete only the snapshot (blob itself is retained) await blob_client.delete_blob(delete_snapshots="only") # [END create_blob_snapshot] # Delete container await blob_service_client.delete_container("containerformyblobsasync")
async def main(forminfo) -> str: result = forminfo.get("result") path = forminfo.get("path") # Get blob name and container from path container = path.split("/")[0] blob = "/".join(path.split("/")[1:]) # Download blob from blob storage blob_service_client = BlobServiceClient.from_connection_string( os.environ["StorageAccount"]) async with blob_service_client: filename = blob + ".json" blob_container_client = blob_service_client.get_container_client( "output") await blob_container_client.upload_blob( filename, json.dumps(result), content_settings=ContentSettings(content_type="application/json"), ) return f"output/{blob}!"
async def main(): try: CONNECTION_STRING = os.environ['AZURE_STORAGE_CONNECTION_STRING'] except KeyError: print("AZURE_STORAGE_CONNECTION_STRING must be set.") sys.exit(1) status = None blob_service_client = BlobServiceClient.from_connection_string( CONNECTION_STRING) source_blob = "http://www.gutenberg.org/files/59466/59466-0.txt" copied_blob = blob_service_client.get_blob_client("mycontainer", '59466-0.txt') # Copy started" await copied_blob.start_copy_from_url(source_blob) for i in range(10): props = await copied_blob.get_blob_properties() status = props.copy.status print("Copy status: " + status) if status == "success": # copy finished break time.sleep(10) if status != "success": # if not finished after 100s, cancel the operation props = await copied_blob.get_blob_properties() print(props.copy.status) copy_id = props.copy.id await copied_blob.abort_copy(copy_id) props = copied_blob.get_blob_properties() print(props.copy.status)
async def _test_acquire_lease_on_blob_async(self): # Instantiate a BlobServiceClient using a connection string from azure.storage.blob.aio import BlobServiceClient blob_service_client = BlobServiceClient.from_connection_string( self.connection_string) # Instantiate a ContainerClient container_client = blob_service_client.get_container_client( "leasemyblobscontainerasync") # Create new Container await container_client.create_container() # Upload a blob to the container with open(SOURCE_FILE, "rb") as data: await container_client.upload_blob(name="my_blob", data=data) # [START acquire_lease_on_blob] # Get the blob client blob_client = blob_service_client.get_blob_client( "leasemyblobscontainerasync", "my_blob") # Acquire a lease on the blob lease = await blob_client.acquire_lease() # Delete blob by passing in the lease await blob_client.delete_blob(lease=lease) # [END acquire_lease_on_blob] # Delete container await blob_service_client.delete_container("leasemyblobscontainerasync" )
async def _test_set_metadata_on_container_async(self): # Instantiate a BlobServiceClient using a connection string from azure.storage.blob.aio import BlobServiceClient blob_service_client = BlobServiceClient.from_connection_string( self.connection_string) # Instantiate a ContainerClient container_client = blob_service_client.get_container_client( "mymetadatacontainerasync") try: # Create new Container await container_client.create_container() # [START set_container_metadata] # Create key, value pairs for metadata metadata = {'type': 'test'} # Set metadata on the container await container_client.set_container_metadata(metadata=metadata) # [END set_container_metadata] # Get container properties properties = (await container_client.get_container_properties()).metadata assert properties == metadata finally: # Delete container await container_client.delete_container()
async def process_blob_files(self, src_container, dest_container, local_temp_dir): try: self.init(local_temp_dir) # Create the BlobServiceClient object which will be used to create a container client blob_service_client = BlobServiceClient.from_connection_string( self.conn_str) async with blob_service_client: container_client = blob_service_client.get_container_client( src_container) tasks = [] async for blob in container_client.list_blobs(): blob_client = container_client.get_blob_client(blob.name) local_file = "{}/{}".format(local_temp_dir, os.path.basename(blob.name)) tasks.append( self.process_blob_file(blob_client, dest_container, local_file)) await asyncio.gather(*tasks) self.cleanup(local_temp_dir) except Exception as e: print(repr(e)) sys.exit(1)
def __init__(self, app): self._app = app if "azureBlobClient" in app: if "token_expiration" in app: # TBD - does this apply for Azure? # check that our token is not about to expire expiration = app["token_expiration"] now = datetime.datetime.now() delta = expiration - now if delta.total_seconds() > 10: self._client = app["azureBlobClient"] return # otherwise, fall through and get a new token log.info("Azure access token has expired - renewing") else: self._client = app["azureBlobClient"] return # first time setup of Azure client or limited time token has expired # TBD - what do do about region? log.info("AzureBlobClient init") azure_connection_string = config.get('azure_connection_string') if not azure_connection_string: msg="No connection string specified" log.error(msg) raise ValueError(msg) log.info(f"Using azure_connection_string: {azure_connection_string}") self._client = BlobServiceClient.from_connection_string(azure_connection_string) app['azureBlobClient'] = self._client # save so same client can be returned in subsequent calls
async def _test_list_blobs_in_container_async(self): # Instantiate a BlobServiceClient using a connection string from azure.storage.blob.aio import BlobServiceClient blob_service_client = BlobServiceClient.from_connection_string( self.connection_string) # Instantiate a ContainerClient container_client = blob_service_client.get_container_client( "myblobscontainerasync") # Create new Container await container_client.create_container() # [START upload_blob_to_container] with open(SOURCE_FILE, "rb") as data: blob_client = await container_client.upload_blob(name="blobby", data=data) properties = await blob_client.get_blob_properties() # [END upload_blob_to_container] # [START list_blobs_in_container] blobs_list = [] async for blob in container_client.list_blobs(): blobs_list.append(blob) # [END list_blobs_in_container] assert blobs_list is not None # Delete container await container_client.delete_container()
def from_connection_string( cls, connection_string: str, container_name: str) -> "AzureBlobAsyncFileStore": return cls( BlobServiceClient.from_connection_string( connection_string), # type: ignore container_name, )
async def reset(): storage = BlobServiceClient.from_connection_string( BLOB_STORAGE_SETTINGS.connection_string ) try: await storage.delete_container(BLOB_STORAGE_SETTINGS.container_name) except ResourceNotFoundError: pass
async def container_access_policy_async(self): # Instantiate a BlobServiceClient using a connection string from azure.storage.blob.aio import BlobServiceClient blob_service_client = BlobServiceClient.from_connection_string( self.connection_string) async with blob_service_client: # Instantiate a ContainerClient container_client = blob_service_client.get_container_client( "myaccesscontainerasync") try: # Create new Container await container_client.create_container() # [START set_container_access_policy] # Create access policy from azure.storage.blob import AccessPolicy, ContainerSasPermissions access_policy = AccessPolicy( permission=ContainerSasPermissions(read=True), expiry=datetime.utcnow() + timedelta(hours=1), start=datetime.utcnow() - timedelta(minutes=1)) identifiers = {'test': access_policy} # Set the access policy on the container await container_client.set_container_access_policy( signed_identifiers=identifiers) # [END set_container_access_policy] # [START get_container_access_policy] policy = await container_client.get_container_access_policy() # [END get_container_access_policy] # [START generate_sas_token] # Use access policy to generate a sas token from azure.storage.blob import generate_container_sas sas_token = generate_container_sas( container_client.account_name, container_client.container_name, account_key=container_client.credential.account_key, policy_id='my-access-policy-id') # [END generate_sas_token] # Use the sas token to authenticate a new client # [START create_container_client_sastoken] from azure.storage.blob.aio import ContainerClient container = ContainerClient.from_container_url( container_url= "https://account.blob.core.windows.net/mycontainerasync", credential=sas_token, ) # [END create_container_client_sastoken] finally: # Delete container await container_client.delete_container()
async def blob_service_stats_async(self): # Instantiate a BlobServiceClient using a connection string from azure.storage.blob.aio import BlobServiceClient blob_service_client = BlobServiceClient.from_connection_string( self.connection_string) # [START get_blob_service_stats] stats = await blob_service_client.get_service_stats()
async def _test_container_access_policy_async(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # Instantiate a BlobServiceClient using a connection string from azure.storage.blob.aio import BlobServiceClient blob_service_client = BlobServiceClient.from_connection_string( self.connection_string) # Instantiate a ContainerClient container_client = blob_service_client.get_container_client( "myaccesscontainerasync") try: # Create new Container await container_client.create_container() # [START set_container_access_policy] # Create access policy from azure.storage.blob import AccessPolicy, ContainerSasPermissions access_policy = AccessPolicy( permission=ContainerSasPermissions(read=True), expiry=datetime.utcnow() + timedelta(hours=1), start=datetime.utcnow() - timedelta(minutes=1)) identifiers = {'test': access_policy} # Set the access policy on the container await container_client.set_container_access_policy( signed_identifiers=identifiers) # [END set_container_access_policy] # [START get_container_access_policy] policy = await container_client.get_container_access_policy() # [END get_container_access_policy] # [START generate_sas_token] # Use access policy to generate a sas token sas_token = container_client.generate_shared_access_signature( policy_id='my-access-policy-id') # [END generate_sas_token] # Use the sas token to authenticate a new client # [START create_container_client_sastoken] from azure.storage.blob.aio import ContainerClient container = ContainerClient.from_container_url( container_url= "https://account.blob.core.windows.net/mycontainerasync", credential=sas_token, ) # [END create_container_client_sastoken] finally: # Delete container await container_client.delete_container()
async def get_storage_account_information_async(self): # Instantiate a BlobServiceClient using a connection string from azure.storage.blob.aio import BlobServiceClient blob_service_client = BlobServiceClient.from_connection_string( self.connection_string) # [START get_blob_service_account_info] account_info = await blob_service_client.get_account_information() print('Using Storage SKU: {}'.format(account_info['sku_name']))
async def main(): try: async with BlobServiceClient.from_connection_string( CONNECTION_STRING) as service_client: containers = service_client.list_containers() async for container in containers: await walk_container(service_client, container) except Exception as error: print(error) sys.exit(1)
def __init__(self, arguments): super().__init__(arguments) connection_string = self.get_from_env("AZURE_STORAGE_CONNECTION_STRING") kwargs = {} kwargs['max_single_put_size'] = self.args.max_put_size kwargs['max_block_size'] = self.args.max_block_size kwargs['min_large_block_upload_threshold'] = self.args.buffer_threshold if not _ServiceTest.service_client or self.args.no_client_share: _ServiceTest.service_client = SyncBlobServiceClient.from_connection_string(conn_str=connection_string, **kwargs) _ServiceTest.async_service_client = AsyncBlobServiceClient.from_connection_string(conn_str=connection_string, **kwargs) self.service_client = _ServiceTest.service_client self.async_service_client =_ServiceTest.async_service_client
async def upload_decompressed_files(self, dest_container, decompression_dir, metadata_json): blob_service_client = BlobServiceClient.from_connection_string( self.conn_str) files_to_upload = self.get_files_to_upload(decompression_dir) async with blob_service_client: for file in files_to_upload: await self.upload_decompressed_file(blob_service_client, file, dest_container, metadata_json)
async def create_container(connection_string: str, container_name: str) -> ContainerClient: """ Creates and initialize a container; returns the client needed to access it. """ blob_service_client = BlobServiceClient.from_connection_string( connection_string) logger.info(f'{"Initializing storage client for account:"}' + f"{blob_service_client.account_name}") container_client = blob_service_client.get_container_client(container_name) await create_container_using_client(container_client) return container_client
async def stream_block_blob(self): import uuid # Instantiate a new BlobServiceClient using a connection string - set chunk size to 1MB from azure.storage.blob import BlobBlock from azure.storage.blob.aio import BlobServiceClient blob_service_client = BlobServiceClient.from_connection_string( self.connection_string, max_single_get_size=1024 * 1024, max_chunk_get_size=1024 * 1024) async with blob_service_client: # Instantiate a new ContainerClient container_client = blob_service_client.get_container_client( "containerasync") # Generate 4MB of data data = b'a' * 4 * 1024 * 1024 try: # Create new Container in the service await container_client.create_container() # Instantiate a new source blob client source_blob_client = container_client.get_blob_client( "source_blob") # Upload content to block blob await source_blob_client.upload_blob(data, blob_type="BlockBlob") destination_blob_client = container_client.get_blob_client( "destination_blob") # This returns a StorageStreamDownloader. stream = await source_blob_client.download_blob() block_list = [] # Read data in chunks to avoid loading all into memory at once async for chunk in stream.chunks(): # process your data (anything can be done here really. `chunk` is a byte array). block_id = str(uuid.uuid4()) await destination_blob_client.stage_block( block_id=block_id, data=chunk) block_list.append(BlobBlock(block_id=block_id)) # Upload the whole chunk to azure storage and make up one blob await destination_blob_client.commit_block_list(block_list) finally: # Delete container await container_client.delete_container()
def __init__(self, blobonedge_module_name, blob_account_name, blob_account_key, sound_container_name, edge_id): localblob_connectionstring = 'DefaultEndpointsProtocol=http;BlobEndpoint=http://' + blobonedge_module_name + ':11002/' + blob_account_name + ';AccountName=' + blob_account_name + ';AccountKey=' + blob_account_key + ';' print("Try to connect to blob on edge by " + localblob_connectionstring) self.blobServiceClient = BlobServiceClient.from_connection_string( localblob_connectionstring) print("Connected to blob on edge") self.containerClient = self.blobServiceClient.get_container_client( sound_container_name) self.soundContainerName = sound_container_name self.edgeId = edge_id