def form_data_af_event_hub_pull(
event: EventData,
get_metadata=False,
):
"""
Used to extract the message from an EventData object. (Event Hub PULL variant.)
:param event: EventData object from which the payload will be extracted from.
:param get_metadata: Flag determining whether to retrieve metadata from the properties attribute.
:return: A JSON-formatted dictionary containing the "DATA" and (if extracted) "METADATA" fields.
"""
payload = event.body_as_str()
metadata = None
# For some unknown the official Microsoft documentation does not disclose that, when
# manually retrieving messages from the Event Hub, it encodes both the keys and values
# of custom properties into bytes, unlike its binding variation.
if get_metadata:
metadata = event.properties
if metadata:
metadata = {
k.decode("utf-8"): metadata[k].decode("utf-8")
for k in metadata
}
return form_store(payload, metadata)
async def test_example_eventhub_async_send_and_receive(live_eventhub_config):
# [START create_eventhub_client_async]
from azure.eventhub.aio import EventHubClient
import os
connection_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format(
os.environ['EVENT_HUB_HOSTNAME'],
os.environ['EVENT_HUB_SAS_POLICY'],
os.environ['EVENT_HUB_SAS_KEY'],
os.environ['EVENT_HUB_NAME'])
client = EventHubClient.from_connection_string(connection_str)
# [END create_eventhub_client_async]
from azure.eventhub import EventData, EventPosition
# [START create_eventhub_client_async_sender]
client = EventHubClient.from_connection_string(connection_str)
# Create an async producer.
producer = client.create_producer(partition_id="0")
# [END create_eventhub_client_async_sender]
# [START create_eventhub_client_async_receiver]
client = EventHubClient.from_connection_string(connection_str)
# Create an async consumer.
receiver = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition('@latest'))
# Create an exclusive async consumer.
receiver = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition('@latest'), owner_level=1)
# [END create_eventhub_client_async_receiver]
client = EventHubClient.from_connection_string(connection_str)
producer = client.create_producer(partition_id="0")
consumer = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition('@latest'))
await consumer.receive(timeout=1)
# [START eventhub_client_async_send]
async with producer:
event_data = EventData(b"A single event")
await producer.send(event_data)
# [END eventhub_client_async_send]
await asyncio.sleep(1)
# [START eventhub_client_async_receive]
logger = logging.getLogger("azure.eventhub")
async with consumer:
received = await consumer.receive(timeout=5)
for event_data in received:
logger.info("Message received:{}".format(event_data.body_as_str()))
# [END eventhub_client_async_receive]
assert len(received) > 0
assert received[0].body_as_str() == "A single event"
assert list(received[-1].body)[0] == b"A single event"
def test_constructor(test_input, expected_result):
if isinstance(expected_result, type):
with pytest.raises(expected_result):
EventData(test_input)
else:
event_data = EventData(test_input)
assert event_data.body_as_str() == expected_result
assert event_data.partition_key is None
assert len(event_data.properties) == 0
assert event_data.enqueued_time is None
assert event_data.offset is None
assert event_data.sequence_number is None
assert len(event_data.system_properties) == 0
with pytest.raises(TypeError):
event_data.body_as_json()
def test_constructor(test_input, expected_result):
if isinstance(expected_result, type):
with pytest.raises(expected_result):
EventData(test_input)
else:
event_data = EventData(test_input)
assert event_data.body_as_str() == expected_result
assert event_data.partition_key is None
assert len(event_data.properties) == 0
assert event_data.enqueued_time is None
assert event_data.offset is None
assert event_data.sequence_number is None
assert len(event_data.system_properties) == 0
assert str(event_data) == "{{ body: '{}', properties: {{}} }}".format(expected_result)
assert repr(event_data) == "EventData(body='{}', properties={{}}, offset=None, sequence_number=None, partition_key=None, enqueued_time=None)".format(expected_result)
with pytest.raises(TypeError):
event_data.body_as_json()
def process_event(self, event: EventData, error_queue: queue.Queue) -> None:
"""Process an event from event hub
Uses process_message to process the event body.
If an error occors it will be put into the queue.
Args:
event: Event to be processed.
error_queue: queue for occuring errors to be handled by calling process
"""
logger.debug(
f"{self.__hash__()}: Processing event with sn {event.sequence_number} and offset {event.offset}"
)
try:
self.process_message(event.body_as_str())
except Exception as e:
error_queue.put([event, e])
def test_example_eventhub_sync_send_and_receive(live_eventhub_config):
# [START create_eventhub_client_connstr]
import os
from azure.eventhub import EventHubClient
connection_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format(
os.environ['EVENT_HUB_HOSTNAME'],
os.environ['EVENT_HUB_SAS_POLICY'],
os.environ['EVENT_HUB_SAS_KEY'],
os.environ['EVENT_HUB_NAME'])
client = EventHubClient.from_connection_string(connection_str)
# [END create_eventhub_client_connstr]
from azure.eventhub import EventData, EventPosition
# [START create_eventhub_client_sender]
client = EventHubClient.from_connection_string(connection_str)
# Create a producer.
producer = client.create_producer(partition_id="0")
# [END create_eventhub_client_sender]
# [START create_eventhub_client_receiver]
client = EventHubClient.from_connection_string(connection_str)
# Create a consumer.
consumer = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition('@latest'))
# Create an exclusive consumer object.
exclusive_consumer = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition("-1"), owner_level=1)
# [END create_eventhub_client_receiver]
client = EventHubClient.from_connection_string(connection_str)
producer = client.create_producer(partition_id="0")
consumer = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition('@latest'))
try:
consumer.receive(timeout=1)
# [START create_event_data]
event_data = EventData("String data")
event_data = EventData(b"Bytes data")
event_data = EventData([b"A", b"B", b"C"])
list_data = ['Message {}'.format(i) for i in range(10)]
event_data = EventData(body=list_data)
# [END create_event_data]
# [START eventhub_client_sync_create_batch]
event_data_batch = producer.create_batch(max_size=10000)
while True:
try:
event_data_batch.try_add(EventData('Message inside EventBatchData'))
except ValueError:
# The EventDataBatch object reaches its max_size.
# You can send the full EventDataBatch object and create a new one here.
break
# [END eventhub_client_sync_create_batch]
# [START eventhub_client_sync_send]
with producer:
event_data = EventData(b"A single event")
producer.send(event_data)
# [END eventhub_client_sync_send]
time.sleep(1)
# [START eventhub_client_sync_receive]
with consumer:
logger = logging.getLogger("azure.eventhub")
received = consumer.receive(timeout=5, max_batch_size=1)
for event_data in received:
logger.info("Message received:{}".format(event_data.body_as_str()))
# [END eventhub_client_sync_receive]
assert len(received) > 0
assert received[0].body_as_str() == "A single event"
assert list(received[-1].body)[0] == b"A single event"
finally:
pass
async def test_example_eventhub_async_send_and_receive(live_eventhub_config):
# [START create_eventhub_client_async]
from azure.eventhub import EventHubClientAsync
import os
connection_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format(
os.environ['EVENT_HUB_HOSTNAME'],
os.environ['EVENT_HUB_SAS_POLICY'],
os.environ['EVENT_HUB_SAS_KEY'],
os.environ['EVENT_HUB_NAME'])
client = EventHubClientAsync.from_connection_string(connection_str)
# [END create_eventhub_client_async]
from azure.eventhub import EventData, Offset
# [START create_eventhub_client_async_sender]
client = EventHubClientAsync.from_connection_string(connection_str)
# Add a async sender to the async client object.
sender = client.add_async_sender(partition="0")
# [END create_eventhub_client_async_sender]
# [START create_eventhub_client_async_receiver]
client = EventHubClientAsync.from_connection_string(connection_str)
# Add a async receiver to the async client object.
receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest'))
# [END create_eventhub_client_async_receiver]
# [START create_eventhub_client_async_epoch_receiver]
client = EventHubClientAsync.from_connection_string(connection_str)
# Add a async receiver to the async client object.
epoch_receiver = client.add_async_epoch_receiver(consumer_group="$default", partition="0", epoch=42)
# [END create_eventhub_client_async_epoch_receiver]
# [START eventhub_client_run_async]
client = EventHubClientAsync.from_connection_string(connection_str)
# Add AsyncSenders/AsyncReceivers
try:
# Opens the connection and starts running all AsyncSender/AsyncReceiver clients.
await client.run_async()
# Start sending and receiving
except:
raise
finally:
await client.stop_async()
# [END eventhub_client_run_async]
client = EventHubClientAsync.from_connection_string(connection_str)
sender = client.add_async_sender(partition="0")
receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest'))
try:
# Opens the connection and starts running all AsyncSender/AsyncReceiver clients.
await client.run_async()
# [START eventhub_client_async_send]
event_data = EventData(b"A single event")
await sender.send(event_data)
# [END eventhub_client_async_send]
time.sleep(1)
# [START eventhub_client_async_receive]
logger = logging.getLogger("azure.eventhub")
received = await receiver.receive(timeout=5)
for event_data in received:
logger.info("Message received:{}".format(event_data.body_as_str()))
# [END eventhub_client_async_receive]
assert len(received) == 1
assert received[0].body_as_str() == "A single event"
assert list(received[-1].body)[0] == b"A single event"
except:
raise
finally:
await client.stop_async()
# [START eventhub_client_async_stop]
client = EventHubClientAsync.from_connection_string(connection_str)
# Add AsyncSenders/AsyncReceivers
try:
# Opens the connection and starts running all AsyncSender/AsyncReceiver clients.
await client.run_async()
# Start sending and receiving
except:
raise
finally:
await client.stop_async()
def test_example_eventhub_sync_send_and_receive(live_eventhub_config):
# [START create_eventhub_client_connstr]
import os
from azure.eventhub import EventHubClient
connection_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format(
os.environ['EVENT_HUB_HOSTNAME'],
os.environ['EVENT_HUB_SAS_POLICY'],
os.environ['EVENT_HUB_SAS_KEY'],
os.environ['EVENT_HUB_NAME'])
client = EventHubClient.from_connection_string(connection_str)
# [END create_eventhub_client_connstr]
from azure.eventhub import EventData, Offset
# [START create_eventhub_client_sender]
client = EventHubClient.from_connection_string(connection_str)
# Add a sender to the client object.
sender = client.add_sender(partition="0")
# [END create_eventhub_client_sender]
# [START create_eventhub_client_receiver]
client = EventHubClient.from_connection_string(connection_str)
# Add a receiver to the client object.
receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest'))
# [END create_eventhub_client_receiver]
# [START create_eventhub_client_epoch_receiver]
client = EventHubClient.from_connection_string(connection_str)
# Add a receiver to the client object with an epoch value.
epoch_receiver = client.add_epoch_receiver(consumer_group="$default", partition="0", epoch=42)
# [END create_eventhub_client_epoch_receiver]
# [START eventhub_client_run]
client = EventHubClient.from_connection_string(connection_str)
# Add Senders/Receivers
try:
client.run()
# Start sending and receiving
except:
raise
finally:
client.stop()
# [END eventhub_client_run]
client = EventHubClient.from_connection_string(connection_str)
sender = client.add_sender(partition="0")
receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest'))
try:
# Opens the connection and starts running all Sender/Receiver clients.
client.run()
# Start sending and receiving
# [START create_event_data]
event_data = EventData("String data")
event_data = EventData(b"Bytes data")
event_data = EventData([b"A", b"B", b"C"])
def batched():
for i in range(10):
yield "Batch data, Event number {}".format(i)
event_data = EventData(batch=batched())
# [END create_event_data]
# [START eventhub_client_sync_send]
event_data = EventData(b"A single event")
sender.send(event_data)
# [END eventhub_client_sync_send]
time.sleep(1)
# [START eventhub_client_sync_receive]
logger = logging.getLogger("azure.eventhub")
received = receiver.receive(timeout=5, max_batch_size=1)
for event_data in received:
logger.info("Message received:{}".format(event_data.body_as_str()))
# [END eventhub_client_sync_receive]
assert len(received) == 1
assert received[0].body_as_str() == "A single event"
assert list(received[-1].body)[0] == b"A single event"
except:
raise
finally:
client.stop()
# [START eventhub_client_stop]
client = EventHubClient.from_connection_string(connection_str)
# Add Senders/Receivers
try:
client.run()
# Start sending and receiving
except:
raise
finally:
client.stop()
def test_example_eventhub_sync_send_and_receive(live_eventhub_config):
# [START create_eventhub_client_connstr]
import os
from azure.eventhub import EventHubClient
connection_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format(
os.environ['EVENT_HUB_HOSTNAME'], os.environ['EVENT_HUB_SAS_POLICY'],
os.environ['EVENT_HUB_SAS_KEY'], os.environ['EVENT_HUB_NAME'])
client = EventHubClient.from_connection_string(connection_str)
# [END create_eventhub_client_connstr]
from azure.eventhub import EventData, Offset
# [START create_eventhub_client_sender]
client = EventHubClient.from_connection_string(connection_str)
# Add a sender to the client object.
sender = client.add_sender(partition="0")
# [END create_eventhub_client_sender]
# [START create_eventhub_client_receiver]
client = EventHubClient.from_connection_string(connection_str)
# Add a receiver to the client object.
receiver = client.add_receiver(consumer_group="$default",
partition="0",
offset=Offset('@latest'))
# [END create_eventhub_client_receiver]
# [START create_eventhub_client_epoch_receiver]
client = EventHubClient.from_connection_string(connection_str)
# Add a receiver to the client object with an epoch value.
epoch_receiver = client.add_epoch_receiver(consumer_group="$default",
partition="0",
epoch=42)
# [END create_eventhub_client_epoch_receiver]
# [START eventhub_client_run]
client = EventHubClient.from_connection_string(connection_str)
# Add Senders/Receivers
try:
client.run()
# Start sending and receiving
except:
raise
finally:
client.stop()
# [END eventhub_client_run]
client = EventHubClient.from_connection_string(connection_str)
sender = client.add_sender(partition="0")
receiver = client.add_receiver(consumer_group="$default",
partition="0",
offset=Offset('@latest'))
try:
# Opens the connection and starts running all Sender/Receiver clients.
client.run()
# Start sending and receiving
# [START create_event_data]
event_data = EventData("String data")
event_data = EventData(b"Bytes data")
event_data = EventData([b"A", b"B", b"C"])
def batched():
for i in range(10):
yield "Batch data, Event number {}".format(i)
event_data = EventData(batch=batched())
# [END create_event_data]
# [START eventhub_client_sync_send]
event_data = EventData(b"A single event")
sender.send(event_data)
# [END eventhub_client_sync_send]
time.sleep(1)
# [START eventhub_client_sync_receive]
logger = logging.getLogger("azure.eventhub")
received = receiver.receive(timeout=5, max_batch_size=1)
for event_data in received:
logger.info("Message received:{}".format(event_data.body_as_str()))
# [END eventhub_client_sync_receive]
assert len(received) == 1
assert received[0].body_as_str() == "A single event"
assert list(received[-1].body)[0] == b"A single event"
except:
raise
finally:
client.stop()
# [START eventhub_client_stop]
client = EventHubClient.from_connection_string(connection_str)
# Add Senders/Receivers
try:
client.run()
# Start sending and receiving
except:
raise
finally:
client.stop()
async def test_example_eventhub_async_send_and_receive(live_eventhub_config):
# [START create_eventhub_client_async]
from azure.eventhub import EventHubClientAsync
import os
connection_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format(
os.environ['EVENT_HUB_HOSTNAME'], os.environ['EVENT_HUB_SAS_POLICY'],
os.environ['EVENT_HUB_SAS_KEY'], os.environ['EVENT_HUB_NAME'])
client = EventHubClientAsync.from_connection_string(connection_str)
# [END create_eventhub_client_async]
from azure.eventhub import EventData, Offset
# [START create_eventhub_client_async_sender]
client = EventHubClientAsync.from_connection_string(connection_str)
# Add a async sender to the async client object.
sender = client.add_async_sender(partition="0")
# [END create_eventhub_client_async_sender]
# [START create_eventhub_client_async_receiver]
client = EventHubClientAsync.from_connection_string(connection_str)
# Add a async receiver to the async client object.
receiver = client.add_async_receiver(consumer_group="$default",
partition="0",
offset=Offset('@latest'))
# [END create_eventhub_client_async_receiver]
# [START create_eventhub_client_async_epoch_receiver]
client = EventHubClientAsync.from_connection_string(connection_str)
# Add a async receiver to the async client object.
epoch_receiver = client.add_async_epoch_receiver(consumer_group="$default",
partition="0",
epoch=42)
# [END create_eventhub_client_async_epoch_receiver]
# [START eventhub_client_run_async]
client = EventHubClientAsync.from_connection_string(connection_str)
# Add AsyncSenders/AsyncReceivers
try:
# Opens the connection and starts running all AsyncSender/AsyncReceiver clients.
await client.run_async()
# Start sending and receiving
except:
raise
finally:
await client.stop_async()
# [END eventhub_client_run_async]
client = EventHubClientAsync.from_connection_string(connection_str)
sender = client.add_async_sender(partition="0")
receiver = client.add_async_receiver(consumer_group="$default",
partition="0",
offset=Offset('@latest'))
try:
# Opens the connection and starts running all AsyncSender/AsyncReceiver clients.
await client.run_async()
# [START eventhub_client_async_send]
event_data = EventData(b"A single event")
await sender.send(event_data)
# [END eventhub_client_async_send]
time.sleep(1)
# [START eventhub_client_async_receive]
logger = logging.getLogger("azure.eventhub")
received = await receiver.receive(timeout=5)
for event_data in received:
logger.info("Message received:{}".format(event_data.body_as_str()))
# [END eventhub_client_async_receive]
assert len(received) == 1
assert received[0].body_as_str() == "A single event"
assert list(received[-1].body)[0] == b"A single event"
except:
raise
finally:
await client.stop_async()
# [START eventhub_client_async_stop]
client = EventHubClientAsync.from_connection_string(connection_str)
# Add AsyncSenders/AsyncReceivers
try:
# Opens the connection and starts running all AsyncSender/AsyncReceiver clients.
await client.run_async()
# Start sending and receiving
except:
raise
finally:
await client.stop_async()