def form_data_af_event_hub_pull( event: EventData, get_metadata=False, ): """ Used to extract the message from an EventData object. (Event Hub PULL variant.) :param event: EventData object from which the payload will be extracted from. :param get_metadata: Flag determining whether to retrieve metadata from the properties attribute. :return: A JSON-formatted dictionary containing the "DATA" and (if extracted) "METADATA" fields. """ payload = event.body_as_str() metadata = None # For some unknown the official Microsoft documentation does not disclose that, when # manually retrieving messages from the Event Hub, it encodes both the keys and values # of custom properties into bytes, unlike its binding variation. if get_metadata: metadata = event.properties if metadata: metadata = { k.decode("utf-8"): metadata[k].decode("utf-8") for k in metadata } return form_store(payload, metadata)
async def test_example_eventhub_async_send_and_receive(live_eventhub_config): # [START create_eventhub_client_async] from azure.eventhub.aio import EventHubClient import os connection_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format( os.environ['EVENT_HUB_HOSTNAME'], os.environ['EVENT_HUB_SAS_POLICY'], os.environ['EVENT_HUB_SAS_KEY'], os.environ['EVENT_HUB_NAME']) client = EventHubClient.from_connection_string(connection_str) # [END create_eventhub_client_async] from azure.eventhub import EventData, EventPosition # [START create_eventhub_client_async_sender] client = EventHubClient.from_connection_string(connection_str) # Create an async producer. producer = client.create_producer(partition_id="0") # [END create_eventhub_client_async_sender] # [START create_eventhub_client_async_receiver] client = EventHubClient.from_connection_string(connection_str) # Create an async consumer. receiver = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition('@latest')) # Create an exclusive async consumer. receiver = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition('@latest'), owner_level=1) # [END create_eventhub_client_async_receiver] client = EventHubClient.from_connection_string(connection_str) producer = client.create_producer(partition_id="0") consumer = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition('@latest')) await consumer.receive(timeout=1) # [START eventhub_client_async_send] async with producer: event_data = EventData(b"A single event") await producer.send(event_data) # [END eventhub_client_async_send] await asyncio.sleep(1) # [START eventhub_client_async_receive] logger = logging.getLogger("azure.eventhub") async with consumer: received = await consumer.receive(timeout=5) for event_data in received: logger.info("Message received:{}".format(event_data.body_as_str())) # [END eventhub_client_async_receive] assert len(received) > 0 assert received[0].body_as_str() == "A single event" assert list(received[-1].body)[0] == b"A single event"
def test_constructor(test_input, expected_result): if isinstance(expected_result, type): with pytest.raises(expected_result): EventData(test_input) else: event_data = EventData(test_input) assert event_data.body_as_str() == expected_result assert event_data.partition_key is None assert len(event_data.properties) == 0 assert event_data.enqueued_time is None assert event_data.offset is None assert event_data.sequence_number is None assert len(event_data.system_properties) == 0 with pytest.raises(TypeError): event_data.body_as_json()
def test_constructor(test_input, expected_result): if isinstance(expected_result, type): with pytest.raises(expected_result): EventData(test_input) else: event_data = EventData(test_input) assert event_data.body_as_str() == expected_result assert event_data.partition_key is None assert len(event_data.properties) == 0 assert event_data.enqueued_time is None assert event_data.offset is None assert event_data.sequence_number is None assert len(event_data.system_properties) == 0 assert str(event_data) == "{{ body: '{}', properties: {{}} }}".format(expected_result) assert repr(event_data) == "EventData(body='{}', properties={{}}, offset=None, sequence_number=None, partition_key=None, enqueued_time=None)".format(expected_result) with pytest.raises(TypeError): event_data.body_as_json()
def process_event(self, event: EventData, error_queue: queue.Queue) -> None: """Process an event from event hub Uses process_message to process the event body. If an error occors it will be put into the queue. Args: event: Event to be processed. error_queue: queue for occuring errors to be handled by calling process """ logger.debug( f"{self.__hash__()}: Processing event with sn {event.sequence_number} and offset {event.offset}" ) try: self.process_message(event.body_as_str()) except Exception as e: error_queue.put([event, e])
def test_example_eventhub_sync_send_and_receive(live_eventhub_config): # [START create_eventhub_client_connstr] import os from azure.eventhub import EventHubClient connection_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format( os.environ['EVENT_HUB_HOSTNAME'], os.environ['EVENT_HUB_SAS_POLICY'], os.environ['EVENT_HUB_SAS_KEY'], os.environ['EVENT_HUB_NAME']) client = EventHubClient.from_connection_string(connection_str) # [END create_eventhub_client_connstr] from azure.eventhub import EventData, EventPosition # [START create_eventhub_client_sender] client = EventHubClient.from_connection_string(connection_str) # Create a producer. producer = client.create_producer(partition_id="0") # [END create_eventhub_client_sender] # [START create_eventhub_client_receiver] client = EventHubClient.from_connection_string(connection_str) # Create a consumer. consumer = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition('@latest')) # Create an exclusive consumer object. exclusive_consumer = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition("-1"), owner_level=1) # [END create_eventhub_client_receiver] client = EventHubClient.from_connection_string(connection_str) producer = client.create_producer(partition_id="0") consumer = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition('@latest')) try: consumer.receive(timeout=1) # [START create_event_data] event_data = EventData("String data") event_data = EventData(b"Bytes data") event_data = EventData([b"A", b"B", b"C"]) list_data = ['Message {}'.format(i) for i in range(10)] event_data = EventData(body=list_data) # [END create_event_data] # [START eventhub_client_sync_create_batch] event_data_batch = producer.create_batch(max_size=10000) while True: try: event_data_batch.try_add(EventData('Message inside EventBatchData')) except ValueError: # The EventDataBatch object reaches its max_size. # You can send the full EventDataBatch object and create a new one here. break # [END eventhub_client_sync_create_batch] # [START eventhub_client_sync_send] with producer: event_data = EventData(b"A single event") producer.send(event_data) # [END eventhub_client_sync_send] time.sleep(1) # [START eventhub_client_sync_receive] with consumer: logger = logging.getLogger("azure.eventhub") received = consumer.receive(timeout=5, max_batch_size=1) for event_data in received: logger.info("Message received:{}".format(event_data.body_as_str())) # [END eventhub_client_sync_receive] assert len(received) > 0 assert received[0].body_as_str() == "A single event" assert list(received[-1].body)[0] == b"A single event" finally: pass
async def test_example_eventhub_async_send_and_receive(live_eventhub_config): # [START create_eventhub_client_async] from azure.eventhub import EventHubClientAsync import os connection_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format( os.environ['EVENT_HUB_HOSTNAME'], os.environ['EVENT_HUB_SAS_POLICY'], os.environ['EVENT_HUB_SAS_KEY'], os.environ['EVENT_HUB_NAME']) client = EventHubClientAsync.from_connection_string(connection_str) # [END create_eventhub_client_async] from azure.eventhub import EventData, Offset # [START create_eventhub_client_async_sender] client = EventHubClientAsync.from_connection_string(connection_str) # Add a async sender to the async client object. sender = client.add_async_sender(partition="0") # [END create_eventhub_client_async_sender] # [START create_eventhub_client_async_receiver] client = EventHubClientAsync.from_connection_string(connection_str) # Add a async receiver to the async client object. receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) # [END create_eventhub_client_async_receiver] # [START create_eventhub_client_async_epoch_receiver] client = EventHubClientAsync.from_connection_string(connection_str) # Add a async receiver to the async client object. epoch_receiver = client.add_async_epoch_receiver(consumer_group="$default", partition="0", epoch=42) # [END create_eventhub_client_async_epoch_receiver] # [START eventhub_client_run_async] client = EventHubClientAsync.from_connection_string(connection_str) # Add AsyncSenders/AsyncReceivers try: # Opens the connection and starts running all AsyncSender/AsyncReceiver clients. await client.run_async() # Start sending and receiving except: raise finally: await client.stop_async() # [END eventhub_client_run_async] client = EventHubClientAsync.from_connection_string(connection_str) sender = client.add_async_sender(partition="0") receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) try: # Opens the connection and starts running all AsyncSender/AsyncReceiver clients. await client.run_async() # [START eventhub_client_async_send] event_data = EventData(b"A single event") await sender.send(event_data) # [END eventhub_client_async_send] time.sleep(1) # [START eventhub_client_async_receive] logger = logging.getLogger("azure.eventhub") received = await receiver.receive(timeout=5) for event_data in received: logger.info("Message received:{}".format(event_data.body_as_str())) # [END eventhub_client_async_receive] assert len(received) == 1 assert received[0].body_as_str() == "A single event" assert list(received[-1].body)[0] == b"A single event" except: raise finally: await client.stop_async() # [START eventhub_client_async_stop] client = EventHubClientAsync.from_connection_string(connection_str) # Add AsyncSenders/AsyncReceivers try: # Opens the connection and starts running all AsyncSender/AsyncReceiver clients. await client.run_async() # Start sending and receiving except: raise finally: await client.stop_async()
def test_example_eventhub_sync_send_and_receive(live_eventhub_config): # [START create_eventhub_client_connstr] import os from azure.eventhub import EventHubClient connection_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format( os.environ['EVENT_HUB_HOSTNAME'], os.environ['EVENT_HUB_SAS_POLICY'], os.environ['EVENT_HUB_SAS_KEY'], os.environ['EVENT_HUB_NAME']) client = EventHubClient.from_connection_string(connection_str) # [END create_eventhub_client_connstr] from azure.eventhub import EventData, Offset # [START create_eventhub_client_sender] client = EventHubClient.from_connection_string(connection_str) # Add a sender to the client object. sender = client.add_sender(partition="0") # [END create_eventhub_client_sender] # [START create_eventhub_client_receiver] client = EventHubClient.from_connection_string(connection_str) # Add a receiver to the client object. receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) # [END create_eventhub_client_receiver] # [START create_eventhub_client_epoch_receiver] client = EventHubClient.from_connection_string(connection_str) # Add a receiver to the client object with an epoch value. epoch_receiver = client.add_epoch_receiver(consumer_group="$default", partition="0", epoch=42) # [END create_eventhub_client_epoch_receiver] # [START eventhub_client_run] client = EventHubClient.from_connection_string(connection_str) # Add Senders/Receivers try: client.run() # Start sending and receiving except: raise finally: client.stop() # [END eventhub_client_run] client = EventHubClient.from_connection_string(connection_str) sender = client.add_sender(partition="0") receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) try: # Opens the connection and starts running all Sender/Receiver clients. client.run() # Start sending and receiving # [START create_event_data] event_data = EventData("String data") event_data = EventData(b"Bytes data") event_data = EventData([b"A", b"B", b"C"]) def batched(): for i in range(10): yield "Batch data, Event number {}".format(i) event_data = EventData(batch=batched()) # [END create_event_data] # [START eventhub_client_sync_send] event_data = EventData(b"A single event") sender.send(event_data) # [END eventhub_client_sync_send] time.sleep(1) # [START eventhub_client_sync_receive] logger = logging.getLogger("azure.eventhub") received = receiver.receive(timeout=5, max_batch_size=1) for event_data in received: logger.info("Message received:{}".format(event_data.body_as_str())) # [END eventhub_client_sync_receive] assert len(received) == 1 assert received[0].body_as_str() == "A single event" assert list(received[-1].body)[0] == b"A single event" except: raise finally: client.stop() # [START eventhub_client_stop] client = EventHubClient.from_connection_string(connection_str) # Add Senders/Receivers try: client.run() # Start sending and receiving except: raise finally: client.stop()
def test_example_eventhub_sync_send_and_receive(live_eventhub_config): # [START create_eventhub_client_connstr] import os from azure.eventhub import EventHubClient connection_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format( os.environ['EVENT_HUB_HOSTNAME'], os.environ['EVENT_HUB_SAS_POLICY'], os.environ['EVENT_HUB_SAS_KEY'], os.environ['EVENT_HUB_NAME']) client = EventHubClient.from_connection_string(connection_str) # [END create_eventhub_client_connstr] from azure.eventhub import EventData, Offset # [START create_eventhub_client_sender] client = EventHubClient.from_connection_string(connection_str) # Add a sender to the client object. sender = client.add_sender(partition="0") # [END create_eventhub_client_sender] # [START create_eventhub_client_receiver] client = EventHubClient.from_connection_string(connection_str) # Add a receiver to the client object. receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) # [END create_eventhub_client_receiver] # [START create_eventhub_client_epoch_receiver] client = EventHubClient.from_connection_string(connection_str) # Add a receiver to the client object with an epoch value. epoch_receiver = client.add_epoch_receiver(consumer_group="$default", partition="0", epoch=42) # [END create_eventhub_client_epoch_receiver] # [START eventhub_client_run] client = EventHubClient.from_connection_string(connection_str) # Add Senders/Receivers try: client.run() # Start sending and receiving except: raise finally: client.stop() # [END eventhub_client_run] client = EventHubClient.from_connection_string(connection_str) sender = client.add_sender(partition="0") receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) try: # Opens the connection and starts running all Sender/Receiver clients. client.run() # Start sending and receiving # [START create_event_data] event_data = EventData("String data") event_data = EventData(b"Bytes data") event_data = EventData([b"A", b"B", b"C"]) def batched(): for i in range(10): yield "Batch data, Event number {}".format(i) event_data = EventData(batch=batched()) # [END create_event_data] # [START eventhub_client_sync_send] event_data = EventData(b"A single event") sender.send(event_data) # [END eventhub_client_sync_send] time.sleep(1) # [START eventhub_client_sync_receive] logger = logging.getLogger("azure.eventhub") received = receiver.receive(timeout=5, max_batch_size=1) for event_data in received: logger.info("Message received:{}".format(event_data.body_as_str())) # [END eventhub_client_sync_receive] assert len(received) == 1 assert received[0].body_as_str() == "A single event" assert list(received[-1].body)[0] == b"A single event" except: raise finally: client.stop() # [START eventhub_client_stop] client = EventHubClient.from_connection_string(connection_str) # Add Senders/Receivers try: client.run() # Start sending and receiving except: raise finally: client.stop()
async def test_example_eventhub_async_send_and_receive(live_eventhub_config): # [START create_eventhub_client_async] from azure.eventhub import EventHubClientAsync import os connection_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format( os.environ['EVENT_HUB_HOSTNAME'], os.environ['EVENT_HUB_SAS_POLICY'], os.environ['EVENT_HUB_SAS_KEY'], os.environ['EVENT_HUB_NAME']) client = EventHubClientAsync.from_connection_string(connection_str) # [END create_eventhub_client_async] from azure.eventhub import EventData, Offset # [START create_eventhub_client_async_sender] client = EventHubClientAsync.from_connection_string(connection_str) # Add a async sender to the async client object. sender = client.add_async_sender(partition="0") # [END create_eventhub_client_async_sender] # [START create_eventhub_client_async_receiver] client = EventHubClientAsync.from_connection_string(connection_str) # Add a async receiver to the async client object. receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) # [END create_eventhub_client_async_receiver] # [START create_eventhub_client_async_epoch_receiver] client = EventHubClientAsync.from_connection_string(connection_str) # Add a async receiver to the async client object. epoch_receiver = client.add_async_epoch_receiver(consumer_group="$default", partition="0", epoch=42) # [END create_eventhub_client_async_epoch_receiver] # [START eventhub_client_run_async] client = EventHubClientAsync.from_connection_string(connection_str) # Add AsyncSenders/AsyncReceivers try: # Opens the connection and starts running all AsyncSender/AsyncReceiver clients. await client.run_async() # Start sending and receiving except: raise finally: await client.stop_async() # [END eventhub_client_run_async] client = EventHubClientAsync.from_connection_string(connection_str) sender = client.add_async_sender(partition="0") receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) try: # Opens the connection and starts running all AsyncSender/AsyncReceiver clients. await client.run_async() # [START eventhub_client_async_send] event_data = EventData(b"A single event") await sender.send(event_data) # [END eventhub_client_async_send] time.sleep(1) # [START eventhub_client_async_receive] logger = logging.getLogger("azure.eventhub") received = await receiver.receive(timeout=5) for event_data in received: logger.info("Message received:{}".format(event_data.body_as_str())) # [END eventhub_client_async_receive] assert len(received) == 1 assert received[0].body_as_str() == "A single event" assert list(received[-1].body)[0] == b"A single event" except: raise finally: await client.stop_async() # [START eventhub_client_async_stop] client = EventHubClientAsync.from_connection_string(connection_str) # Add AsyncSenders/AsyncReceivers try: # Opens the connection and starts running all AsyncSender/AsyncReceiver clients. await client.run_async() # Start sending and receiving except: raise finally: await client.stop_async()