bytes_payload = b"".join(b for b in event.body) print('The received bytes of the EventData is {}.'.format(bytes_payload)) # Use the deserialize method to convert bytes to dict object. # The deserialize method would extract the schema id from the payload, and automatically retrieve the Avro Schema # from the Schema Registry Service. The schema would be cached locally for future usage. deserialized_data = avro_serializer.deserialize(bytes_payload) print( 'The dict data after deserialization is {}'.format(deserialized_data)) # create an EventHubConsumerClient instance eventhub_consumer = EventHubConsumerClient.from_connection_string( conn_str=EVENTHUB_CONNECTION_STR, consumer_group='$Default', eventhub_name=EVENTHUB_NAME, ) # create a AvroSerializer instance avro_serializer = AvroSerializer(client=SchemaRegistryClient( fully_qualified_namespace=SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE, credential=DefaultAzureCredential()), group_name=GROUP_NAME, auto_register_schemas=True) try: with eventhub_consumer, avro_serializer: eventhub_consumer.receive( on_event=on_event, starting_position=
import os from azure.eventhub import EventHubConsumerClient from azure.eventhub.extensions.checkpointstoreblob import BlobCheckpointStore CONNECTION_STR = os.environ["EVENT_HUB_CONN_STR"] STORAGE_CONNECTION_STR = os.environ["AZURE_STORAGE_CONN_STR"] BLOB_CONTAINER_NAME = "your-blob-container-name" # Please make sure the blob container resource exists. def on_event(partition_context, event): # Put your code here. # Avoid time-consuming operations. print(event) partition_context.update_checkpoint(event) if __name__ == '__main__': checkpoint_store = BlobCheckpointStore.from_connection_string( STORAGE_CONNECTION_STR, container_name=BLOB_CONTAINER_NAME, ) client = EventHubConsumerClient.from_connection_string( CONNECTION_STR, "$Default", checkpoint_store=checkpoint_store) try: client.receive(on_event) except KeyboardInterrupt: client.close()
print("Received event from partition: {}".format( partition_context.partition_id)) # Put your code here to do some operations on the event. # Avoid time-consuming operations. print(event) partition_context.update_checkpoint(event) if __name__ == '__main__': checkpoint_store = BlobCheckpointStore.from_connection_string( STORAGE_CONNECTION_STR, "eventprocessor") consumer_client = EventHubConsumerClient.from_connection_string( conn_str=CONNECTION_STR, consumer_group='$Default', checkpoint_store= checkpoint_store, # For load-balancing and checkpoint. Leave None for no load-balancing ) try: with consumer_client: """ Without specified partition_id, the receive will try to receive events from all partitions and if provided with partition manager, the client will load-balance partition assignment with other EventHubConsumerClient instances which also try to receive events from all partitions and use the same storage resource. """ consumer_client.receive(on_event=on_event) # With specified partition_id, load-balance will be disabled # client.receive(on_event=on_event, consumer_group='$Default', partition_id='0') except KeyboardInterrupt: print('Stop receiving.')
from azure.eventhub import EventHubConsumerClient import logging ##EVENT HUB DETAILS connection_str = 'Endpoint=sb://streamuckafka.servicebus.windows.net/;SharedAccessKeyName=StreamUCPolicy;SharedAccessKey=C1ds2c1s+nsO0OWk+IlGRfDn3IY2pnWVTrXJ1EF2kMs=' eventhub_name = 'streamuckafkaeventhub' consumer_group = "$default" client = EventHubConsumerClient.from_connection_string( connection_str, consumer_group, eventhub_name=eventhub_name) logger = logging.getLogger("azure.eventhub") logging.basicConfig(level=logging.INFO) def on_event(partition_context, event): logger.info("Received event from partition {}".format( partition_context.partition_id)) logger.info("Message received:{}".format(event)) partition_context.update_checkpoint(event) with client: client.receive(on_event=on_event, starting_position="-1")
EVENT_POSITION = EventPosition("-1") PARTITION = "0" def on_event(partition_context, event): print("Received event from partition {}".format( partition_context.partition_id)) # Put your code here to do some operations on the event. # Avoid time-consuming operations. print(event) print("Last enqueued event properties from partition: {} is: {}".format( partition_context.partition_id, event.last_enqueued_event_properties)) if __name__ == '__main__': consumer_client = EventHubConsumerClient.from_connection_string( conn_str=CONNECTION_STR, event_hub_path=EVENT_HUB, ) try: with consumer_client: consumer_client.receive(on_event=on_event, consumer_group='$Default', partition_id='0', track_last_enqueued_event_properties=True) except KeyboardInterrupt: print('Stop receiving.')
def on_error(partition_context, error): # put your code here print("Partition: {} met an exception during receiving: {}".format( partition_context.partition_id, error)) def on_events(partition_context, event): # put your code here print("Received event from partition: {}".format( partition_context.partition_id)) if __name__ == '__main__': consumer_client = EventHubConsumerClient.from_connection_string( conn_str=CONNECTION_STR, eventhub_name=EVENTHUB_NAME, ) try: with consumer_client: consumer_client.receive( on_events=on_events, consumer_group='$Default', on_partition_initialize=on_partition_initialize, on_partition_close=on_partition_close, on_error=on_error) except KeyboardInterrupt: print('Stop receiving.')
'onePropertyNameIShouldGiveAProperName'] = 'youShouldChangeThis' aux_iot_hub_send_message_to_device( device_name=event_device_name, message_body=command_to_device_message_body, message_properties=command_to_device_message_properties) if __name__ == '__main__': #FIRST CHECK IF THE CONNECTION STRING IS OK aux_validate_connection_string() #CONNECT TO THE IOT HUB BUILT IT ENDPOINT aux_iot_hub_built_in_event_hub_consumer_client = EventHubConsumerClient.from_connection_string( conn_str=AUX_EVENT_HUB_NAMESPACE_CONNECTION_STRING, consumer_group='app', eventhub_name=AUX_EVENTHUB_NAME) MAXIMUM_TEMPERATURE = 38 try: with aux_iot_hub_built_in_event_hub_consumer_client: print("Starting sample temperature monitor application") print("Maximum temperature set to {} degrees".format( MAXIMUM_TEMPERATURE)) #EVERY TIME WE RECEIVE AN EVENT WE CALL THE METHOD ON EVENT aux_iot_hub_built_in_event_hub_consumer_client.receive( on_event=on_iot_hub_message_event, starting_position="@latest") except KeyboardInterrupt: print('Stopped receiving.')
CONNECTION_STR = os.environ["EVENT_HUB_CONN_STR"] STORAGE_CONNECTION_STR = os.environ["AZURE_STORAGE_CONN_STR"] logging.basicConfig(level=logging.INFO) def do_operation(event): # put your code here print(event) def process_events(partition_context, events): # put your code here print("received events: {} from partition: {}".format( len(events), partition_context.partition_id)) for event in events: do_operation(event) partition_context.update_checkpoint(events[-1]) if __name__ == '__main__': container_client = ContainerClient.from_connection_string( STORAGE_CONNECTION_STR, "eventprocessor") partition_manager = BlobPartitionManager(container_client=container_client) client = EventHubConsumerClient.from_connection_string( CONNECTION_STR, partition_manager=partition_manager) try: client.receive(process_events, "$default") except KeyboardInterrupt: client.close()
def test_send_amqp_annotated_message(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubProducerClient.from_connection_string(connection_str) with client: sequence_body = [b'message', 123.456, True] footer = {'footer_key': 'footer_value'} prop = {"subject": "sequence"} seq_app_prop = {"body_type": "sequence"} sequence_message = AmqpAnnotatedMessage( sequence_body=sequence_body, footer=footer, properties=prop, application_properties=seq_app_prop ) value_body = {b"key": [-123, b'data', False]} header = {"priority": 10} anno = {"ann_key": "ann_value"} value_app_prop = {"body_type": "value"} value_message = AmqpAnnotatedMessage( value_body=value_body, header=header, annotations=anno, application_properties=value_app_prop ) data_body = [b'aa', b'bb', b'cc'] data_app_prop = {"body_type": "data"} del_anno = {"delann_key": "delann_value"} data_message = AmqpAnnotatedMessage( data_body=data_body, header=header, delivery_annotations=del_anno, application_properties=data_app_prop ) body_ed = """{"json_key": "json_val"}""" prop_ed = {"raw_prop": "raw_value"} cont_type_ed = "text/plain" corr_id_ed = "corr_id" mess_id_ed = "mess_id" event_data = EventData(body_ed) event_data.content_type = cont_type_ed event_data.correlation_id = corr_id_ed event_data.message_id = mess_id_ed batch = client.create_batch() batch.add(data_message) batch.add(value_message) batch.add(sequence_message) batch.add(event_data) client.send_batch(batch) client.send_batch([data_message, value_message, sequence_message, event_data]) received_count = {} received_count["data_msg"] = 0 received_count["seq_msg"] = 0 received_count["value_msg"] = 0 received_count["normal_msg"] = 0 def check_values(event): raw_amqp_message = event.raw_amqp_message if raw_amqp_message.body_type == AmqpMessageBodyType.DATA: if raw_amqp_message.application_properties and raw_amqp_message.application_properties.get(b'body_type') == b'data': body = [data for data in raw_amqp_message.body] assert data_body == body assert event.body_as_str() == "aabbcc" assert raw_amqp_message.delivery_annotations[b'delann_key'] == b'delann_value' assert raw_amqp_message.application_properties[b'body_type'] == b'data' received_count["data_msg"] += 1 else: assert event.body_as_json() == {'json_key': 'json_val'} assert event.correlation_id == corr_id_ed assert event.message_id == mess_id_ed assert event.content_type == cont_type_ed assert event.body_type == AmqpMessageBodyType.DATA received_count["normal_msg"] += 1 elif raw_amqp_message.body_type == AmqpMessageBodyType.SEQUENCE: body = [sequence for sequence in raw_amqp_message.body] assert [sequence_body] == body assert event.body_as_str() == "['message', 123.456, True]" assert raw_amqp_message.footer[b'footer_key'] == b'footer_value' assert raw_amqp_message.properties.subject == b'sequence' assert raw_amqp_message.application_properties[b'body_type'] == b'sequence' received_count["seq_msg"] += 1 elif raw_amqp_message.body_type == AmqpMessageBodyType.VALUE: assert raw_amqp_message.body == value_body assert event.body_as_str() == "{'key': [-123, 'data', False]}" assert raw_amqp_message.annotations[b'ann_key'] == b'ann_value' assert raw_amqp_message.application_properties[b'body_type'] == b'value' received_count["value_msg"] += 1 def on_event(partition_context, event): on_event.received.append(event) on_event.received = [] client = EventHubConsumerClient.from_connection_string(connection_str, consumer_group='$default') with client: thread = threading.Thread(target=client.receive, args=(on_event,), kwargs={"starting_position": "-1"}) thread.start() time.sleep(15) for event in on_event.received: check_values(event) assert len(on_event.received) == 8 assert received_count["data_msg"] == 2 assert received_count["seq_msg"] == 2 assert received_count["value_msg"] == 2 assert received_count["normal_msg"] == 2
PARTITION = "0" HTTP_PROXY = { 'proxy_hostname': '127.0.0.1', # proxy hostname 'proxy_port': 3128, # proxy port 'username': '******', # username used for proxy authentication if needed 'password': '******' # password used for proxy authentication if needed } def on_event(partition_context, event): print("received event from partition: {}".format(partition_context.partition_id)) # do some operations on the event print(event) consumer_client = EventHubConsumerClient.from_connection_string( conn_str=CONNECTION_STR, consumer_group='$Default', eventhub_name=EVENTHUB_NAME, http_proxy=HTTP_PROXY) producer_client = EventHubProducerClient.from_connection_string( conn_str=CONNECTION_STR, eventhub_name=EVENTHUB_NAME, http_proxy=HTTP_PROXY) with producer_client: event_data_batch = producer_client.create_batch(max_size_in_bytes=10000) while True: try: event_data_batch.add(EventData('Message inside EventBatchData')) except ValueError: # EventDataBatch object reaches max_size. # New EventDataBatch object can be created here to send more data break producer_client.send_batch(event_data_batch) print('Finish sending.')
import asyncio from app import app from app.constants import Config from azure.eventhub import EventHubConsumerClient client = EventHubConsumerClient.from_connection_string( Config.EVENT_HUB_CONNNECTION_STRING, Config.CONSUMER_GROUP, eventhub_name=Config.EVENT_HUB_NAME) def on_event(partition_context, event): from app.loggers import context # context.do_logging("ID: {} Message: {}".format( # partition_context.partition_id, event.body_as_str(encoding='UTF-8'))) partition_context.update_checkpoint(event) def receive_data(): with client: client.receive( on_event=on_event, # "-1" is from the beginning of the partition. starting_position="-1", ) receive_data()
from azure.eventhub.extensions.checkpointstoreblob import BlobCheckpointStore CONNECTION_STR = os.environ["EVENT_HUB_CONN_STR"] EVENTHUB_NAME = os.environ['EVENT_HUB_NAME'] STORAGE_CONNECTION_STR = os.environ["AZURE_STORAGE_CONN_STR"] BLOB_CONTAINER_NAME = "your-blob-container-name" # Please make sure the blob container resource exists. def on_event(partition_context, event): # Put your code here. # Avoid time-consuming operations. print(event) partition_context.update_checkpoint(event) if __name__ == '__main__': checkpoint_store = BlobCheckpointStore.from_connection_string( STORAGE_CONNECTION_STR, container_name=BLOB_CONTAINER_NAME, ) client = EventHubConsumerClient.from_connection_string( CONNECTION_STR, consumer_group='$Default', eventhub_name=EVENTHUB_NAME, checkpoint_store=checkpoint_store) try: client.receive(on_event) except KeyboardInterrupt: client.close()
import os import sys import time from azure.eventhub import EventHubConsumerClient, EventHubProducerClient # Connection String CONN = sys.argv[1] CONSUMER_GROUP = "$Default" client = EventHubConsumerClient.from_connection_string(CONN, CONSUMER_GROUP) def on_event(partition_context, event): print("event: {}", event) print("Received event from partition {}".format( partition_context.partition_id)) partition_context.update_checkpoint(event) with client: client.receive( on_event=on_event, starting_position="-1", # "-1" is from the beginning of the partition. )
partition_context.partition_id)) # Put your code here to do some operations on the event. # Avoid time-consuming operations. print(event) partition_context.update_checkpoint(event) if __name__ == '__main__': container_client = ContainerClient.from_connection_string( STORAGE_CONNECTION_STR, "eventprocessor") partition_manager = BlobPartitionManager(container_client) consumer_client = EventHubConsumerClient.from_connection_string( conn_str=CONNECTION_STR, partition_manager= partition_manager, # For load balancing and checkpoint. Leave None for no load balancing ) try: with consumer_client: """ Without specified partition_id, the receive will try to receive events from all partitions and if provided with partition manager, the client will load-balance partition assignment with other EventHubConsumerClient instances which also try to receive events from all partitions and use the same storage resource. """ consumer_client.receive(on_event=on_event, consumer_group='$Default') # With specified partition_id, load-balance will be disabled # client.receive(on_event=on_event, consumer_group='$Default', partition_id='0') except KeyboardInterrupt: