def _create_container_client(self):
return ContainerClient.from_connection_string(
self.__conn_string,
self.__container_name,
logging_enable=False,
max_single_get_size=2 * 1024 * 1024,
max_chunk_get_size=2 * 1024 * 1024)
async def main():
container_client = ContainerClient.from_connection_string(STORAGE_CONNECTION_STR, "eventprocessor")
partition_manager = BlobPartitionManager(container_client)
client = EventHubConsumerClient.from_connection_string(
CONNECTION_STR,
partition_manager=partition_manager, # For load balancing and checkpoint. Leave None for no load balancing
)
async with client:
await receive(client)
def __enter__(self):
self._client = EventHubClient.from_connection_string(
self._eh_connection_string)
self._storage_client = ContainerClient.from_connection_string(
conn_str=self._sa_connection_string,
container=self._sa_container_name)
self._partition_manager = BlobPartitionManager(self._storage_client)
self._event_processor = EventProcessor(self._client, "$default",
self._partition_processor,
self._partition_manager)
async def main():
try:
CONNECTION_STRING = "xxxx"
except KeyError:
print("AZURE_STORAGE_CONNECTION_STRING must be set.")
sys.exit(1)
container = ContainerClient.from_connection_string(CONNECTION_STRING, container_name="containerName")
path="example/data/"
async with container:
async for blob in container.list_blobs(name_starts_with=path):
print(blob.name + '\n')
def __init__(self):
connection_string = os.environ.get("STORAGE_CONNECTION_STRING")
if not connection_string:
raise Exception(
"Undefined environment variable STORAGE_CONNECTION_STRING")
self.container_client = SyncContainerClient.from_connection_string(
conn_str=connection_string, container_name=self.container_name)
#TODO: I really hate this.
self.async_container_client = AsyncContainerClient.from_connection_string(
conn_str=connection_string, container_name=self.container_name)
async def main():
try:
CONNECTION_STRING = os.environ['AZURE_STORAGE_CONNECTION_STRING']
except KeyError:
print("AZURE_STORAGE_CONNECTION_STRING must be set.")
sys.exit(1)
container = ContainerClient.from_connection_string(
CONNECTION_STRING, container_name="mycontainer")
async for blob in container.list_blobs():
print(blob.name + '\n')
async def main():
eventhub_client = EventHubClient.from_connection_string(
EVENT_HUB_CONNECTION_STR, receive_timeout=5, retry_total=3)
storage_container_client = ContainerClient.from_connection_string(
STORAGE_CONTAINER_CONNECTION_STR, STORAGE_CONTAINER_PATH)
partition_manager = BlobPartitionManager(
storage_container_client) # use the BlobPartitonManager to save
event_processor = EventProcessor(eventhub_client, "mllab-consumers-group",
MyPartitionProcessor, partition_manager)
print("Starting...")
async with storage_container_client:
asyncio.ensure_future(event_processor.start())
await asyncio.sleep(20) # run for a while
await event_processor.stop()
def download(paths):
now = lambda: time.time()
start = now()
tasks = []
for path in paths:
container_client = ContainerClient.from_connection_string(
conn_str=os.getenv("AZURE_STORAGE_CONNECTION_STRING"),
container_name=os.getenv("CONTAINER_NAME"))
tasks.append(downloadPath(container_client, path))
if tasks:
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))
print('TIME: ', now() - start)
async def get_blob_service_client_from_container_client_async(self):
# Instantiate a BlobServiceClient using a connection string
from azure.storage.blob.aio import ContainerClient
container_client1 = ContainerClient.from_connection_string(
self.connection_string, "container")
await container_client1.create_container()
# [START get_blob_service_client_from_container_client]
blob_service_client = container_client1.get_blob_service_client()
print(await blob_service_client.get_service_properties())
container_client2 = blob_service_client.get_container_client(
"container")
print(await container_client2.get_container_properties())
await container_client2.delete_container()
await container_client1.close()
def get_live_storage_blob_client():
try:
storage_connection_str = os.environ['AZURE_STORAGE_CONN_STR']
except KeyError:
return None, None
try:
from azure.storage.blob import BlobServiceClient
from azure.storage.blob.aio import ContainerClient
except ImportError or ModuleNotFoundError:
return None, None
container_str = str(uuid.uuid4())
blob_service_client = BlobServiceClient.from_connection_string(
storage_connection_str)
blob_service_client.create_container(container_str)
container_client = ContainerClient.from_connection_string(
storage_connection_str, container_str)
return container_str, container_client
async def GlobalSetupAsync(self):
connection_string = os.environ.get("STORAGE_CONNECTION_STRING")
if not connection_string:
raise Exception(
"Undefined environment variable STORAGE_CONNECTION_STRING")
type(self
).container_client = SyncContainerClient.from_connection_string(
conn_str=connection_string,
container_name=self.container_name)
type(
self
).async_container_client = AsyncContainerClient.from_connection_string(
conn_str=connection_string, container_name=self.container_name)
await type(self).async_container_client.__aenter__()
type(self).container_client.create_container()
data = b'a' * self.Arguments.size
type(self).container_client.upload_blob(self.blob_name, data)
async def _test_auth_connection_string_async(self):
# [START auth_from_connection_string]
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(self.connection_string)
# [END auth_from_connection_string]
# [START auth_from_connection_string_container]
from azure.storage.blob.aio import ContainerClient
container_client = ContainerClient.from_connection_string(
self.connection_string, container_name="mycontainer")
# [END auth_from_connection_string_container]
# [START auth_from_connection_string_blob]
from azure.storage.blob.aio import BlobClient
blob_client = BlobClient.from_connection_string(
self.connection_string, container_name="mycontainer", blob_name="blobname.txt")
# [END auth_from_connection_string_blob]
# Get account information for the Blob Service
account_info = await blob_service_client.get_account_information()
assert account_info is not None
async def _upload_blob() -> None:
async with ContainerClient.from_connection_string( # type: ignore[attr-defined]
_connection_string(subscription, resource_group, storage_name),
container_name,
) as container_client:
class UploadTasks:
def __init__(self,
paths_to_upload: List[pathlib.Path],
max_open_files: int = 100):
self.index = 0
self.paths_to_upload = paths_to_upload
self.semaphore = asyncio.Semaphore(max_open_files)
# Forward reference not possible on Python 3.6:
def __iter__(self): # type: ignore[no-untyped-def]
return self
def __next__(self) -> asyncio.Task:
try:
task = asyncio.create_task(
_upload_file(
container_client,
self.paths_to_upload[self.index],
source_folder,
self.semaphore,
))
except IndexError as exc:
raise StopIteration from exc
self.index += 1
return task
def __len__(self) -> int:
return len(self.paths_to_upload)
for task in tqdm.as_completed(
UploadTasks(paths_to_upload),
bar_format="{l_bar} {bar} | Uploaded {n_fmt}/{total_fmt}",
):
await task
def from_connection_string(
cls, conn_str, container_name, *, credential=None, **kwargs
):
# type: (str, str, Optional[Any], str) -> BlobCheckpointStore
"""Create BlobCheckpointStore from a storage connection string.
:param str conn_str:
A connection string to an Azure Storage account.
:param container_name:
The container name for the blobs.
:type container_name: str
:param credential:
The credentials with which to authenticate. This is optional if the
account URL already has a SAS token, or the connection string already has shared
access key values. The value can be a SAS token string, an account shared access
key, or an instance of a TokenCredentials class from azure.identity.
Credentials provided here will take precedence over those in the connection string.
"""
container_client = ContainerClient.from_connection_string(
conn_str, container_name, credential=credential, **kwargs
)
return cls(None, None, container_client=container_client)
STORAGE_CONNECTION_STR = os.environ["AZURE_STORAGE_CONN_STR"]
logging.basicConfig(level=logging.INFO)
async def do_operation(event):
# put your code here
# do some sync or async operations. If the operation is i/o intensive, async will have better performance
print(event)
async def process_events(partition_context, events):
# put your code here
await asyncio.gather(*[do_operation(event) for event in events])
await partition_context.update_checkpoint(events[-1])
if __name__ == '__main__':
loop = asyncio.get_event_loop()
container_client = ContainerClient.from_connection_string(
STORAGE_CONNECTION_STR, "eventprocessor")
partition_manager = BlobPartitionManager(container_client=container_client)
client = EventHubConsumerClient.from_connection_string(
CONNECTION_STR, partition_manager=partition_manager)
try:
loop.run_until_complete(client.receive(process_events, "$default"))
except KeyboardInterrupt:
loop.run_until_complete(client.close())
finally:
loop.stop()
def _create_container_client(self):
return ContainerClient.from_connection_string(self.__conn_string,
self.__container_name,
logging_enable=False)
async def get_db() -> Any:
db = Session()
# Make PostgreSQL return float8 columns with highest precision. If we don't
# do this, we may lose up to 3 of the least significant digits.
if IS_POSTGRES:
db.execute("SET extra_float_digits=3")
try:
yield db
db.commit()
db.close()
except:
db.rollback()
db.close()
raise
if HAS_AZURE_BLOB_STORAGE:
import asyncio
from azure.core.exceptions import ResourceNotFoundError
from azure.storage.blob.aio import ContainerClient
azure_blob_container = ContainerClient.from_connection_string(
os.environ[ENV_BLOB], BLOB_CONTAINER)
async def create_container_if_not_exist() -> None:
try:
await azure_blob_container.get_container_properties()
except ResourceNotFoundError:
await azure_blob_container.create_container()
