async def main():
# Connection string for namespace and name for event hub
VAR_CONN_STR = "Connection string for namespace"
VAR_EVENTHUB_NAME = "Name for event hub"
# Consumer group name
VAR_CONSUMER_GROUP = "$Default"
# Connectionstring for storage account and blob container name
VAR_STORAGE_CONN_STR = "Connectionstring for storage account"
VAR_BLOB_CONTAINER = "Blob container name"
# Storage service API version for Azure Stack Hub
STORAGE_SERVICE_API_VERSION = "2017-11-09"
# Create an Azure blob checkpoint store to store the checkpoints.
checkpoint_store = BlobCheckpointStore.from_connection_string(
VAR_STORAGE_CONN_STR,
container_name=VAR_BLOB_CONTAINER,
api_version=STORAGE_SERVICE_API_VERSION)
# Create a consumer client for the event hub.
client = EventHubConsumerClient.from_connection_string(
conn_str=VAR_CONN_STR,
consumer_group=VAR_CONSUMER_GROUP,
eventhub_name=VAR_EVENTHUB_NAME,
checkpoint_store=checkpoint_store)
async with client:
# Call the receive method.
await client.receive(on_event=on_event)
async def _claim_and_list_ownership(connection_str, container_name):
fully_qualified_namespace = 'test_namespace'
eventhub_name = 'eventhub'
consumer_group = '$default'
ownership_cnt = 8
checkpoint_store = BlobCheckpointStore.from_connection_string(
connection_str, container_name)
async with checkpoint_store:
ownership_list = await checkpoint_store.list_ownership(
fully_qualified_namespace=fully_qualified_namespace,
eventhub_name=eventhub_name,
consumer_group=consumer_group)
assert len(ownership_list) == 0
ownership_list = []
for i in range(ownership_cnt):
ownership = {}
ownership['fully_qualified_namespace'] = fully_qualified_namespace
ownership['eventhub_name'] = eventhub_name
ownership['consumer_group'] = consumer_group
ownership['owner_id'] = 'ownerid'
ownership['partition_id'] = str(i)
ownership['last_modified_time'] = time.time()
ownership_list.append(ownership)
await checkpoint_store.claim_ownership(ownership_list)
ownership_list = await checkpoint_store.list_ownership(
fully_qualified_namespace=fully_qualified_namespace,
eventhub_name=eventhub_name,
consumer_group=consumer_group)
assert len(ownership_list) == ownership_cnt
async def main():
# Create an Azure blob checkpoint store to store the checkpoints.
checkpoint_store = BlobCheckpointStore.from_connection_string(checkpoint_conn_str, checkpoint_container_name)
# Create a consumer client for the event hub.
client = EventHubConsumerClient.from_connection_string(conn_str, consumer_group="$Default", eventhub_name=event_hub_name, checkpoint_store=checkpoint_store)
client_future = client.receive(on_event=on_event, starting_position=-1)
print('client is listening')
producer = EventHubProducerClient.from_connection_string(conn_str=conn_str, eventhub_name=event_hub_name)
async with producer:
# Create a batch.
event_data_batch = await producer.create_batch()
# Add events to the batch.
event_data_batch.add(EventData('First event '))
event_data_batch.add(EventData('Second event'))
event_data_batch.add(EventData('Third event'))
# Send the batch of events to the event hub.
await producer.send_batch(event_data_batch)
print('sent events')
await client_future
async def begin_processing(
self,
processed_messages: List[int],
):
"""
The function the concurrent tasks will be running.
Starts the shared checkpoint blob and the Event Hub client.
:param processed_messages: Counter of processed events.
:return:
"""
# Initialize the checkpoint store mechanism.
checkpoint_store = BlobCheckpointStore.from_connection_string(
self.checkpoint_conn_str,
self.checkpoint_container,
)
# Initialize the asynchronous Consumer Client.
client = EventHubConsumerClient.from_connection_string(
conn_str=self.hub_connection_str,
consumer_group=self.consumer_group,
eventhub_name=self.hub_name,
checkpoint_store=checkpoint_store,
idle_timeout=self.idle_timeout,
)
# Begin the pull.
# (See parent class.)
await self.time_constrained_pull(
client,
processed_messages,
)
async def _update_checkpoint(connection_str, container_name):
fully_qualified_namespace = 'test_namespace'
eventhub_name = 'eventhub'
consumer_group = '$default'
partition_cnt = 8
checkpoint_store = BlobCheckpointStore.from_connection_string(
connection_str, container_name)
async with checkpoint_store:
for i in range(partition_cnt):
checkpoint = {
'fully_qualified_namespace': fully_qualified_namespace,
'eventhub_name': eventhub_name,
'consumer_group': consumer_group,
'partition_id': str(i),
'offset': '2',
'sequence_number': 20
}
await checkpoint_store.update_checkpoint(checkpoint)
checkpoint_list = await checkpoint_store.list_checkpoints(
fully_qualified_namespace=fully_qualified_namespace,
eventhub_name=eventhub_name,
consumer_group=consumer_group)
assert len(checkpoint_list) == partition_cnt
for checkpoint in checkpoint_list:
assert checkpoint['offset'] == '2'
assert checkpoint['sequence_number'] == 20
def create_client(args):
if args.storage_conn_str:
checkpoint_store = BlobCheckpointStore.from_connection_string(
args.storage_conn_str, args.storage_container_name)
else:
checkpoint_store = None
transport_type = TransportType.Amqp if args.transport_type == 0 else TransportType.AmqpOverWebsocket
http_proxy = None
if args.proxy_hostname:
http_proxy = {
"proxy_hostname": args.proxy_hostname,
"proxy_port": args.proxy_port,
"username": args.proxy_username,
"password": args.proxy_password,
}
if args.conn_str:
client = EventHubConsumerClientTest.from_connection_string(
args.conn_str,
args.consumer_group,
eventhub_name=args.eventhub,
checkpoint_store=checkpoint_store,
load_balancing_interval=args.load_balancing_interval,
auth_timeout=args.auth_timeout,
http_proxy=http_proxy,
transport_type=transport_type,
logging_enable=args.uamqp_logging_enable)
elif args.hostname:
client = EventHubConsumerClientTest(
fully_qualified_namespace=args.hostname,
eventhub_name=args.eventhub,
consumer_group=args.consumer_group,
credential=EventHubSharedKeyCredential(args.sas_policy,
args.sas_key),
checkpoint_store=checkpoint_store,
load_balancing_interval=args.load_balancing_interval,
auth_timeout=args.auth_timeout,
http_proxy=http_proxy,
transport_type=transport_type,
logging_enable=args.uamqp_logging_enable)
elif args.aad_client_id:
credential = ClientSecretCredential(args.tenant_id, args.aad_client_id,
args.aad_secret)
client = EventHubConsumerClientTest(
fully_qualified_namespace=args.hostname,
eventhub_name=args.eventhub,
consumer_group=args.consumer_group,
credential=credential,
checkpoint_store=checkpoint_store,
load_balancing_interval=args.load_balancing_interval,
auth_timeout=args.auth_timeout,
http_proxy=http_proxy,
transport_type=transport_type,
logging_enable=args.uamqp_logging_enable)
return client
async def main():
# Create an Azure blob checkpoint store to store the checkpoints.
checkpoint_store = BlobCheckpointStore.from_connection_string("" ,"ehubcontainer")
# Create a consumer client for the event hub.
client = EventHubConsumerClient.from_connection_string("Endpoint=", consumer_group="$Default", eventhub_name="ehpolux", checkpoint_store=checkpoint_store)
async with client:
# Call the receive method. Read from the beginning of the partition (starting_position: "-1")
await client.receive(on_event=on_event, starting_position="-1")
async def main():
checkpoint_store = BlobCheckpointStore.from_connection_string(STORAGE_CONNECTION_STR, BLOB_CONTAINER_NAME)
client = EventHubConsumerClient.from_connection_string(
CONNECTION_STR,
consumer_group="$Default",
checkpoint_store=checkpoint_store, # For load-balancing and checkpoint. Leave None for no load-balancing.
)
async with client:
await receive(client)
async def main():
# Create an Azure blob checkpoint store to store the checkpoints.
checkpoint_store = BlobCheckpointStore.from_connection_string("AZURE STORAGE CONNECTION STRING", "BLOB CONTAINER NAME")
# Create a consumer client for the event hub.
client = EventHubConsumerClient.from_connection_string("EVENT HUBS NAMESPACE CONNECTION STRING", consumer_group="$Default", eventhub_name="EVENT HUB NAME", checkpoint_store=checkpoint_store)
async with client:
# Call the receive method. Read from the beginning of the partition (starting_position: "-1")
await client.receive(on_event=on_event, starting_position="-1")
async def main():
# Create an Azure blob checkpoint store to store the checkpoints.
Checkpoint_store = BlobCheckpointStore.from_connection_string(
"DefaultEndpointsProtocol=https;AccountName=pycomstorage;AccountKey=uZRtTT/f4Wvxoy50FJIs+AzRA3ut7arVXCIP8MLo25jJnWlAx3h7QfD0EVvgNwpcqcAqz8mHr0iJMQ3wjCm0Bw==;EndpointSuffix=core.windows.net",
"temperatureblobstorage")
# Create a consumer client for the event hub.
client = EventHubConsumerClient.from_connection_string(
"Endpoint=sb://realtimetemp.servicebus.windows.net/;SharedAccessKeyName=RealTimeTemp_TempOutput_policy;SharedAccessKey=1KMNBdrnuWM0LGAz4ruFiV/OdZ9s4VZdcIIJWQmnqcM=;EntityPath=realtimetempoutput",
consumer_group="$Default",
eventhub_name="realtimetempoutput")
async with client:
# Call the receive method. Read from the beginning of the partition (starting_position: "-1")
await client.receive(on_event=on_event, starting_position="-1")
async def main():
# Create an Azure blob checkpoint store to store the checkpoints.
checkpoint_store = BlobCheckpointStore.from_connection_string("DefaultEndpointsProtocol=https;AccountName=storagestreamingg;AccountKey=J+CI/bRN7fe1lFP8azsPcluMgIkySFfJdMdezunMzqU6XKjoTm7ixMrGiQMiOjrcaVl2TKAQCyaE9vMztPrpYA==;EndpointSuffix=core.windows.net",\
"streamingstoragee")
# Create a consumer client for the event hub.
client = EventHubConsumerClient.from_connection_string("Endpoint=sb://eventhubss1.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=dVONFNgFZXiMbGsoWQX9YSzp7q0AZ3f3jxrTPCK/iEc=",\
consumer_group="$Default",\
eventhub_name="firsttopic",\
checkpoint_store=checkpoint_store)
async with client:
# Call the receive method. Read from the beginning of the partition (starting_position: "-1")
await client.receive(on_event=on_event, starting_position="-1")
def ii_init(self, record_info_in: Sdk.RecordInfo) -> bool:
self.EventField = record_info_in.get_field_by_name('Event', throw_error=False)
if self.EventField is None:
self.parent.display_error_msg("Incoming data source must contain an 'Event' text field that pushes 'Start' and 'End' events")
return False
self.parent.Output.init(self.RecordInfo)
checkpoint_store = BlobCheckpointStore.from_connection_string(self.parent.CheckpointConnStr,
self.parent.CheckpointContainer)
self.Client = EventHubConsumerClient.from_connection_string(self.parent.EventHubsConnStr,
consumer_group=self.parent.ConsumerGroup,
eventhub_name=self.parent.EventHubName,
checkpoint_store=checkpoint_store)
self.parent.display_info_msg("Event Hubs receive client created")
return True
async def main():
# create an Azure blob checkpoint store to store the checkpoints
checkpoint_store = BlobCheckpointStore.from_connection_string(
config["storageAccountConnStr"], config["storageAccountName"])
# create a consumer client for the event hub
client = EventHubConsumerClient.from_connection_string(
conn_str=config["eventHubConnStr"],
consumer_group=config["consumerGroup"],
eventhub_name=config["eventHubName"],
checkpoint_store=checkpoint_store)
async with client:
# call the receive method
await client.receive(on_event=on_event)
async def main():
# Create an Azure blob checkpoint store to store the checkpoints.
checkpoint_store = BlobCheckpointStore.from_connection_string(AZ_CS, BLOB)
# Create a consumer client for the event hub.
client = EventHubConsumerClient.from_connection_string(
EH_CS,
consumer_group="$Default",
eventhub_name=EH_NAME,
checkpoint_store=checkpoint_store) # noqa
async with client:
# Call the receive method.
await client.receive(on_event=on_event)
async def main():
# Create an Azure blob checkpoint store to store the checkpoints.
Checkpoint_store = BlobCheckpointStore.from_connection_string(
"DefaultEndpointsProtocol=https;AccountName=individstorage;AccountKey=sflIN1b46bYLMOLdDy0G7ssweyIzI5iuHSqSPV+1SsM7i0dbFZagSBt1Bi4GYTCkPFWiWQiCRzSTaSPnK3ttSA==;EndpointSuffix=core.windows.net",
"individstorage")
# Create a consumer client for the event hub.
client = EventHubConsumerClient.from_connection_string(
"Endpoint=sb://kdgslimkippenhok.servicebus.windows.net/;SharedAccessKeyName=tempAnalytics_OutputTime_policy;SharedAccessKey=aQjkTANBDDWOqknzbLbnVwSqDIwlX+tMTsVt5UIJZzE=;EntityPath=skeventhub",
consumer_group="$Default",
eventhub_name="skeventhub")
async with client:
# Call the receive method. Read from the beginning of the partition (starting_position: "-1")
await client.receive(on_event=on_event, starting_position="-1")
async def main():
# Create an Azure blob checkpoint store to store the checkpoints.
checkpoint_store = BlobCheckpointStore.from_connection_string(
"DefaultEndpointsProtocol=https;AccountName=eventhublab;AccountKey=c2djYUaXQDRvz+woOOeQdYrBTwm4jXkz7EXETJ9Fr6sHLMO+VfRqFKiYou1Bp/47Khcxi3BxY6sKsfddddddddTyKC29CPw==;EndpointSuffix=corffdf.windows.net",
"offset")
# Create a consumer client for the event hub.
client = EventHubConsumerClient.from_connection_string(
"Endpoint=sb://eventhublab.servicebus.windows.net/;SharedAccessKeyName=recv;SharedAccessKey=6yJCo5yI2PyNXwmHqCcles+wWlEmxwJFTOuJdffffffffH71LHG8=;EntityPath=demo-eventhub",
consumer_group="$Default",
eventhub_name="demo-eventhub",
checkpoint_store=checkpoint_store)
async with client:
# Call the receive method. Read from the beginning of the partition (starting_position: "-1")
await client.receive(on_event=on_event, starting_position="-1")
async def run(storage_account_connection_string, blob_name,
event_hub_connection_string, event_hub_name):
# Create an Azure blob checkpoint store to store the checkpoints.
checkpoint_store = BlobCheckpointStore.from_connection_string(
storage_account_connection_string, blob_name)
# Create a consumer client for the event hub.
client = EventHubConsumerClient.from_connection_string(
event_hub_connection_string,
consumer_group="$Default",
eventhub_name=event_hub_name,
checkpoint_store=checkpoint_store)
async with client:
# Call the receive method. Read from the beginning of the partition (starting_position: "-1")
await client.receive(on_event=on_event, starting_position="-1")
async def main():
connection_str = "Endpoint=sb://avnish327030.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=BwK0zozJe+Oz46F5BgpHUtq5zkzqMKCzPdc2kOg2WW8="
event_hubname = "waferpredictionevent"
# Create an Azure blob checkpoint store to store the checkpoints.
checkpoint_store = BlobCheckpointStore.from_connection_string(
"DefaultEndpointsProtocol=https;AccountName=predictionlocation;AccountKey=V9VbOgYmDmbYH5ACyNy5vIACtfPXX4P5D4u+nkUnjWpOO32SP4wwRcsfRUefq3Rv94PmSTOZugT7ahtzyyV9Pw==;EndpointSuffix=core.windows.net",
"avnish")
# Create a consumer client for the event hub.
client = EventHubConsumerClient.from_connection_string(connection_str, consumer_group="$Default",
eventhub_name=event_hubname,
checkpoint_store=checkpoint_store)
async with client:
# Call the receive method. Read from the beginning of the partition (starting_position: "-1")
await client.receive(on_event=on_event, starting_position="-1")
async def receive_batch():
checkpoint_store = BlobCheckpointStore.from_connection_string(
STORAGE_CONNECTION_STR, BLOB_CONTAINER_NAME)
client = EventHubConsumerClient.from_connection_string(
CONNECTION_STR,
consumer_group="$Default",
eventhub_name=EVENTHUB_NAME,
checkpoint_store=checkpoint_store,
)
async with client:
await client.receive_batch(
on_event_batch=on_event_batch,
max_batch_size=100,
starting_position=
"-1", # "-1" is from the beginning of the partition.
)
async def main():
env_eventhub_connection = os.getenv('ehubNamespaceConnectionString')
env_eventhub_name = os.getenv('eventhubName')
env_storageConnectionString = os.getenv('storageConnectionString')
env_storageContainerName = os.getenv('storageContainerName')
# Create an Azure blob checkpoint store to store the checkpoints.
checkpoint_store = BlobCheckpointStore.from_connection_string(
env_storageConnectionString, env_storageContainerName)
# Create a consumer client for the event hub.
client = EventHubConsumerClient.from_connection_string(
env_eventhub_connection,
consumer_group="$Default",
eventhub_name=env_eventhub_name,
checkpoint_store=checkpoint_store)
async with client:
# Call the receive method. Read from the beginning of the partition (starting_position: "-1")
await client.receive(on_event=on_event, starting_position="-1")
async def consume(config: ConsumerConfig, delegate: MessageStorageDelegate):
"""
Setup and start a message topic consumer and storage delegate.
:param config: A ConsumerConfig object
:param delegate: A Storage delegate object
:return: None
"""
# Create a consumer client for the event hub.
logger.info(f"Consuming {config}")
if config.checkpoint_store_conn_str and config.checkpoint_store_container_name:
# Use an azure blob storage container to store position within partition
checkpoint_store = BlobCheckpointStore.from_connection_string(config.checkpoint_store_conn_str,
config.checkpoint_store_container_name)
client = EventHubConsumerClient(
fully_qualified_namespace=config.fully_qualified_namespace,
consumer_group=config.consumer_group,
eventhub_name=config.topic,
credential=EventHubSharedKeyCredential(config.shared_access_policy, config.key),
checkpoint_store=checkpoint_store
)
else:
client = EventHubConsumerClient(
fully_qualified_namespace=config.fully_qualified_namespace,
consumer_group=config.consumer_group,
eventhub_name=config.topic,
credential=EventHubSharedKeyCredential(config.shared_access_policy, config.key)
)
handler = MessageHandler(
storage_delegate=delegate,
buffer_size=config.buffer_size,
max_buffer_time_in_sec=config.max_buffer_time_in_seconds,
max_time_to_keep_data_in_seconds=config.max_time_to_keep_data_in_seconds,
data_eviction_interval_in_seconds=config.data_eviction_interval_in_seconds,
checkpoint_after_messages=config.checkpoint_after_messages
)
async with client:
await client.receive(on_event=handler.received_event,
on_error=errored,
on_partition_close=partition_closed,
on_partition_initialize=partition_initialized,
starting_position=-1)
async def _update_checkpoint(live_storage_blob_client):
fully_qualified_namespace = 'test_namespace'
eventhub_name = 'eventhub'
consumer_group = '$default'
partition_cnt = 8
async with live_storage_blob_client:
checkpoint_store = BlobCheckpointStore(
container_client=live_storage_blob_client)
for i in range(partition_cnt):
await checkpoint_store.update_checkpoint(fully_qualified_namespace,
eventhub_name,
consumer_group, str(i),
'2', 20)
checkpoint_list = await checkpoint_store.list_checkpoints(
fully_qualified_namespace=fully_qualified_namespace,
eventhub_name=eventhub_name,
consumer_group=consumer_group)
assert len(checkpoint_list) == partition_cnt
for checkpoint in checkpoint_list:
assert checkpoint['offset'] == '2'
assert checkpoint['sequence_number'] == '20'
STORAGE_SERVICE_API_VERSION = "2017-11-09"
async def on_event(partition_context, event):
# Put your code here.
# Do some sync or async operations. If the operation is i/o intensive, async will have better performance.
print(event)
await partition_context.update_checkpoint(event)
async def main(client):
async with client:
await client.receive(on_event)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
checkpoint_store = BlobCheckpointStore.from_connection_string(
STORAGE_CONNECTION_STR,
container_name=BLOB_CONTAINER_NAME,
api_version=STORAGE_SERVICE_API_VERSION
)
client = EventHubConsumerClient.from_connection_string(
CONNECTION_STR,
"$Default",
checkpoint_store=checkpoint_store
)
try:
loop.run_until_complete(main(client))
finally:
loop.stop()
BLOB_CONTAINER_NAME = "your-blob-container-name" # Please make sure the blob container resource exists.
async def on_event(partition_context, event):
# Put your code here.
# Do some sync or async operations. If the operation is i/o intensive, async will have better performance.
print(event)
await partition_context.update_checkpoint(event)
async def main(client):
async with client:
await client.receive(on_event)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
checkpoint_store = BlobCheckpointStore.from_connection_string(
STORAGE_CONNECTION_STR,
container_name=BLOB_CONTAINER_NAME,
)
client = EventHubConsumerClient.from_connection_string(
CONNECTION_STR,
consumer_group='$Default',
eventhub_name=EVENTHUB_NAME,
checkpoint_store=checkpoint_store
)
try:
loop.run_until_complete(main(client))
finally:
loop.stop()
CONNECTION_STR = os.environ["EVENT_HUB_CONN_STR"]
STORAGE_CONNECTION_STR = os.environ["AZURE_STORAGE_CONN_STR"]
async def on_event(partition_context, event):
# Put your code here.
# Do some sync or async operations. If the operation is i/o intensive, async will have better performance
print(event)
await partition_context.update_checkpoint(event)
async def main(client):
async with client:
await client.receive(on_event)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
checkpoint_store = BlobCheckpointStore.from_connection_string(
STORAGE_CONNECTION_STR,
container_name="eventprocessor"
)
client = EventHubConsumerClient.from_connection_string(
CONNECTION_STR,
"$Default",
checkpoint_store=checkpoint_store
)
try:
loop.run_until_complete(main(client))
finally:
loop.stop()
async def _create_checkpoint_store(self):
self.checkpoint_store = \
BlobCheckpointStore.from_connection_string(
self.checkpoint_store_conn_string,
self.blob_container_name,
)
CONNECTION_STR = os.environ["EVENT_HUB_CONN_STR"]
STORAGE_CONNECTION_STR = os.environ["AZURE_STORAGE_CONN_STR"]
logging.basicConfig(level=logging.INFO)
async def do_operation(event):
# put your code here
# do some sync or async operations. If the operation is i/o intensive, async will have better performance
print(event)
async def process_events(partition_context, events):
# put your code here
await asyncio.gather(*[do_operation(event) for event in events])
await partition_context.update_checkpoint(events[-1])
if __name__ == '__main__':
loop = asyncio.get_event_loop()
container_client = ContainerClient.from_connection_string(STORAGE_CONNECTION_STR, "eventprocessor")
checkpoint_store = BlobCheckpointStore(container_client=container_client)
client = EventHubConsumerClient.from_connection_string(CONNECTION_STR, checkpoint_store=checkpoint_store)
try:
loop.run_until_complete(client.receive(process_events, "$default"))
except KeyboardInterrupt:
loop.run_until_complete(client.close())
finally:
loop.stop()