def _update_checkpoint(storage_connection_str, container_name):
fully_qualified_namespace = 'test_namespace'
eventhub_name = 'eventhub'
consumer_group = '$default'
partition_cnt = 8
checkpoint_store = BlobCheckpointStore.from_connection_string(
storage_connection_str, container_name)
with checkpoint_store:
for i in range(partition_cnt):
checkpoint = {
'fully_qualified_namespace': fully_qualified_namespace,
'eventhub_name': eventhub_name,
'consumer_group': consumer_group,
'partition_id': str(i),
'offset': '2',
'sequence_number': 20
}
checkpoint_store.update_checkpoint(checkpoint)
checkpoint_list = checkpoint_store.list_checkpoints(
fully_qualified_namespace=fully_qualified_namespace,
eventhub_name=eventhub_name,
consumer_group=consumer_group)
assert len(checkpoint_list) == partition_cnt
for checkpoint in checkpoint_list:
assert checkpoint['offset'] == '2'
assert checkpoint['sequence_number'] == 20
def create_client(args):
if args.storage_conn_str:
checkpoint_store = BlobCheckpointStore.from_connection_string(
args.storage_conn_str, args.storage_container_name)
else:
checkpoint_store = None
transport_type = TransportType.Amqp if args.transport_type == 0 else TransportType.AmqpOverWebsocket
http_proxy = None
if args.proxy_hostname:
http_proxy = {
"proxy_hostname": args.proxy_hostname,
"proxy_port": args.proxy_port,
"username": args.proxy_username,
"password": args.proxy_password,
}
if args.conn_str:
client = EventHubConsumerClientTest.from_connection_string(
args.conn_str,
args.consumer_group,
eventhub_name=args.eventhub,
checkpoint_store=checkpoint_store,
load_balancing_interval=args.load_balancing_interval,
auth_timeout=args.auth_timeout,
http_proxy=http_proxy,
transport_type=transport_type,
logging_enable=args.uamqp_logging_enable)
elif args.hostname:
client = EventHubConsumerClientTest(
fully_qualified_namespace=args.hostname,
eventhub_name=args.eventhub,
consumer_group=args.consumer_group,
credential=EventHubSharedKeyCredential(args.sas_policy,
args.sas_key),
checkpoint_store=checkpoint_store,
load_balancing_interval=args.load_balancing_interval,
auth_timeout=args.auth_timeout,
http_proxy=http_proxy,
transport_type=transport_type,
logging_enable=args.uamqp_logging_enable)
elif args.aad_client_id:
credential = ClientSecretCredential(args.tenant_id, args.aad_client_id,
args.aad_secret)
client = EventHubConsumerClientTest(
fully_qualified_namespace=args.hostname,
eventhub_name=args.eventhub,
consumer_group=args.consumer_group,
credential=credential,
checkpoint_store=checkpoint_store,
load_balancing_interval=args.load_balancing_interval,
auth_timeout=args.auth_timeout,
http_proxy=http_proxy,
transport_type=transport_type,
logging_enable=args.uamqp_logging_enable)
return client
def receive_batch():
checkpoint_store = BlobCheckpointStore.from_connection_string(STORAGE_CONNECTION_STR, BLOB_CONTAINER_NAME)
client = EventHubConsumerClient.from_connection_string(
CONNECTION_STR,
consumer_group="$Default",
eventhub_name=EVENTHUB_NAME,
checkpoint_store=checkpoint_store,
)
with client:
client.receive_batch(
on_event_batch=on_event_batch,
max_batch_size=100,
starting_position="-1", # "-1" is from the beginning of the partition.
)
def _claim_and_list_ownership(storage_connection_str, container_name):
fully_qualified_namespace = 'test_namespace'
eventhub_name = 'eventhub'
consumer_group = '$default'
ownership_cnt = 8
checkpoint_store = BlobCheckpointStore.from_connection_string(
storage_connection_str, container_name)
with checkpoint_store:
ownership_list = checkpoint_store.list_ownership(
fully_qualified_namespace=fully_qualified_namespace,
eventhub_name=eventhub_name,
consumer_group=consumer_group)
assert len(ownership_list) == 0
ownership_list = []
for i in range(ownership_cnt):
ownership = {}
ownership['fully_qualified_namespace'] = fully_qualified_namespace
ownership['eventhub_name'] = eventhub_name
ownership['consumer_group'] = consumer_group
ownership['owner_id'] = 'ownerid'
ownership['partition_id'] = str(i)
ownership['last_modified_time'] = time.time()
ownership["offset"] = "1"
ownership["sequence_number"] = "1"
ownership_list.append(ownership)
claimed_ownership = checkpoint_store.claim_ownership(ownership_list)
for i in range(ownership_cnt):
assert ownership_list[i]['partition_id'] == str(i)
assert claimed_ownership[i]['partition_id'] == str(i)
assert ownership_list[i] != claimed_ownership[i]
ownership_list = checkpoint_store.list_ownership(
fully_qualified_namespace=fully_qualified_namespace,
eventhub_name=eventhub_name,
consumer_group=consumer_group)
assert len(ownership_list) == ownership_cnt
def on_event(partition_context, event):
# Put your code here.
# Avoid time-consuming operations.
p_id = partition_context.partition_id
print("Received event from partition: {}".format(p_id))
now_time = time.time()
p_id = partition_context.partition_id
last_checkpoint_time = partition_last_checkpoint_time.get(p_id)
if last_checkpoint_time is None or (
now_time - last_checkpoint_time) >= checkpoint_time_interval:
partition_context.update_checkpoint(event)
partition_last_checkpoint_time[p_id] = now_time
if __name__ == '__main__':
checkpoint_store = BlobCheckpointStore.from_connection_string(
STORAGE_CONNECTION_STR, BLOB_CONTAINER_NAME)
consumer_client = EventHubConsumerClient.from_connection_string(
conn_str=CONNECTION_STR,
consumer_group='$Default',
checkpoint_store=
checkpoint_store, # For load-balancing and checkpoint. Leave None for no load-balancing.
)
try:
with consumer_client:
"""
Without specified partition_id, the receive will try to receive events from all partitions and if provided
with a checkpoint store, the client will load-balance partition assignment with other EventHubConsumerClient
instances which also try to receive events from all partitions and use the same storage resource.
"""
consumer_client.receive(
STORAGE_CONNECTION_STR = os.environ["AZURE_STORAGE_CONN_STR"]
def on_event(partition_context, event):
print("Received event from partition: {}".format(
partition_context.partition_id))
# Put your code here to do some operations on the event.
# Avoid time-consuming operations.
print(event)
partition_context.update_checkpoint(event)
if __name__ == '__main__':
checkpoint_store = BlobCheckpointStore.from_connection_string(
STORAGE_CONNECTION_STR, "eventprocessor")
consumer_client = EventHubConsumerClient.from_connection_string(
conn_str=CONNECTION_STR,
consumer_group='$Default',
checkpoint_store=
checkpoint_store, # For load-balancing and checkpoint. Leave None for no load-balancing
)
try:
with consumer_client:
"""
Without specified partition_id, the receive will try to receive events from all partitions and if provided with
partition manager, the client will load-balance partition assignment with other EventHubConsumerClient instances
which also try to receive events from all partitions and use the same storage resource.
"""
consumer_client.receive(on_event=on_event)
import os
from azure.eventhub import EventHubConsumerClient
from azure.eventhub.extensions.checkpointstoreblob import BlobCheckpointStore
CONNECTION_STR = os.environ["EVENT_HUB_CONN_STR"]
STORAGE_CONNECTION_STR = os.environ["AZURE_STORAGE_CONN_STR"]
BLOB_CONTAINER_NAME = "your-blob-container-name" # Please make sure the blob container resource exists.
STORAGE_SERVICE_API_VERSION = "2017-11-09"
def on_event(partition_context, event):
# Put your code here.
# Avoid time-consuming operations.
print(event)
partition_context.update_checkpoint(event)
if __name__ == '__main__':
checkpoint_store = BlobCheckpointStore.from_connection_string(
STORAGE_CONNECTION_STR,
container_name=BLOB_CONTAINER_NAME,
api_version=STORAGE_SERVICE_API_VERSION)
client = EventHubConsumerClient.from_connection_string(
CONNECTION_STR, "$Default", checkpoint_store=checkpoint_store)
try:
client.receive(on_event)
except KeyboardInterrupt:
client.close()
from azure.eventhub.extensions.checkpointstoreblob import BlobCheckpointStore
from azure.storage.blob import ContainerClient
CONNECTION_STR = os.environ["EVENT_HUB_CONN_STR"]
STORAGE_CONNECTION_STR = os.environ["AZURE_STORAGE_CONN_STR"]
logging.basicConfig(level=logging.INFO)
def do_operation(event):
# put your code here
print(event)
def process_events(partition_context, events):
# put your code here
print("received events: {} from partition: {}".format(len(events), partition_context.partition_id))
for event in events:
do_operation(event)
partition_context.update_checkpoint(events[-1])
if __name__ == '__main__':
container_client = ContainerClient.from_connection_string(STORAGE_CONNECTION_STR, "eventprocessor")
checkpoint_store = BlobCheckpointStore(container_client=container_client)
client = EventHubConsumerClient.from_connection_string(CONNECTION_STR, checkpoint_store=checkpoint_store)
try:
client.receive(process_events, "$default")
except KeyboardInterrupt:
client.close()
