def create_eventhub_consumer_client():
# [START create_eventhub_consumer_client_from_conn_str_sync]
import os
from azure.eventhub import EventHubConsumerClient
event_hub_connection_str = os.environ['EVENT_HUB_CONN_STR']
event_hub = os.environ['EVENT_HUB_NAME']
consumer = EventHubConsumerClient.from_connection_string(
conn_str=event_hub_connection_str, event_hub_path=event_hub)
# [END create_eventhub_consumer_client_from_conn_str_sync]
# [START create_eventhub_consumer_client_sync]
import os
from azure.eventhub import EventHubConsumerClient, EventHubSharedKeyCredential
hostname = os.environ['EVENT_HUB_HOSTNAME']
event_hub = os.environ['EVENT_HUB_NAME']
shared_access_policy = os.environ['EVENT_HUB_SAS_POLICY']
shared_access_key = os.environ['EVENT_HUB_SAS_KEY']
consumer = EventHubConsumerClient(host=hostname,
event_hub_path=event_hub,
credential=EventHubSharedKeyCredential(
shared_access_policy,
shared_access_key))
# [END create_eventhub_consumer_client_sync]
return consumer
def example_create_eventhub_consumer_client():
# [START create_eventhub_consumer_client_from_conn_str_sync]
import os
from azure.eventhub import EventHubConsumerClient
event_hub_connection_str = os.environ['EVENT_HUB_CONN_STR']
eventhub_name = os.environ['EVENT_HUB_NAME']
consumer = EventHubConsumerClient.from_connection_string(
conn_str=event_hub_connection_str,
consumer_group='$Default',
eventhub_name=eventhub_name # EventHub name should be specified if it doesn't show up in connection string.
)
# [END create_eventhub_consumer_client_from_conn_str_sync]
# [START create_eventhub_consumer_client_sync]
import os
from azure.eventhub import EventHubConsumerClient, EventHubSharedKeyCredential
fully_qualified_namespace = os.environ['EVENT_HUB_HOSTNAME']
eventhub_name = os.environ['EVENT_HUB_NAME']
shared_access_policy = os.environ['EVENT_HUB_SAS_POLICY']
shared_access_key = os.environ['EVENT_HUB_SAS_KEY']
credential = EventHubSharedKeyCredential(shared_access_policy, shared_access_key)
consumer = EventHubConsumerClient(
fully_qualified_namespace=fully_qualified_namespace,
eventhub_name=eventhub_name,
consumer_group='$Default',
credential=credential)
# [END create_eventhub_consumer_client_sync]
return consumer
def test_receive_load_balancing(connstr_senders):
if sys.platform.startswith('darwin'):
pytest.skip("Skipping on OSX - test code using multiple threads. Sometimes OSX aborts python process")
connection_str, senders = connstr_senders
pm = InMemoryPartitionManager()
client1 = EventHubConsumerClient.from_connection_string(
connection_str, partition_manager=pm, load_balancing_interval=1)
client2 = EventHubConsumerClient.from_connection_string(
connection_str, partition_manager=pm, load_balancing_interval=1)
def on_event(partition_context, event):
pass
with client1, client2:
worker1 = threading.Thread(target=client1.receive,
args=(on_event,),
kwargs={"consumer_group": "$default", "initial_event_position": "-1"})
worker2 = threading.Thread(target=client2.receive,
args=(on_event,),
kwargs={"consumer_group": "$default", "initial_event_position": "-1"})
worker1.start()
worker2.start()
time.sleep(10)
assert len(client1._event_processors[("$default", ALL_PARTITIONS)]._consumers) == 1
assert len(client2._event_processors[("$default", ALL_PARTITIONS)]._consumers) == 1
def test_receive_with_event_position_sync(connstr_senders, position, inclusive,
expected_result):
def on_event(partition_context, event):
assert partition_context.last_enqueued_event_properties.get(
'sequence_number') == event.sequence_number
assert partition_context.last_enqueued_event_properties.get(
'offset') == event.offset
assert partition_context.last_enqueued_event_properties.get(
'enqueued_time') == event.enqueued_time
assert partition_context.last_enqueued_event_properties.get(
'retrieval_time') is not None
if position == "offset":
on_event.event_position = event.offset
elif position == "sequence":
on_event.event_position = event.sequence_number
else:
on_event.event_position = event.enqueued_time
on_event.event = event
on_event.event_position = None
connection_str, senders = connstr_senders
senders[0].send(EventData(b"Inclusive"))
senders[1].send(EventData(b"Inclusive"))
client = EventHubConsumerClient.from_connection_string(
connection_str, consumer_group='$default')
with client:
thread = threading.Thread(target=client.receive,
args=(on_event, ),
kwargs={
"starting_position": "-1",
"starting_position_inclusive": inclusive,
"track_last_enqueued_event_properties":
True
})
thread.daemon = True
thread.start()
time.sleep(10)
assert on_event.event_position is not None
thread.join()
senders[0].send(EventData(expected_result))
senders[1].send(EventData(expected_result))
client2 = EventHubConsumerClient.from_connection_string(
connection_str, consumer_group='$default')
with client2:
thread = threading.Thread(target=client2.receive,
args=(on_event, ),
kwargs={
"starting_position":
on_event.event_position,
"starting_position_inclusive": inclusive,
"track_last_enqueued_event_properties":
True
})
thread.daemon = True
thread.start()
time.sleep(10)
assert on_event.event.body_as_str() == expected_result
thread.join()
def test_receive_owner_level(connstr_senders):
def on_event(partition_context, event):
pass
def on_error(partition_context, error):
on_error.error = error
on_error.error = None
connection_str, senders = connstr_senders
client1 = EventHubConsumerClient.from_connection_string(connection_str, consumer_group='$default')
client2 = EventHubConsumerClient.from_connection_string(connection_str, consumer_group='$default')
with client1, client2:
thread1 = threading.Thread(target=client1.receive, args=(on_event,),
kwargs={"partition_id": "0", "starting_position": "-1",
"on_error": on_error})
thread1.start()
for i in range(5):
ed = EventData("Event Number {}".format(i))
senders[0].send(ed)
time.sleep(10)
thread2 = threading.Thread(target=client2.receive, args=(on_event,),
kwargs = {"partition_id": "0", "starting_position": "-1", "owner_level": 1})
thread2.start()
for i in range(5):
ed = EventData("Event Number {}".format(i))
senders[0].send(ed)
time.sleep(20)
thread1.join()
thread2.join()
assert isinstance(on_error.error, EventHubError)
def test_get_partition_ids(live_eventhub):
client = EventHubConsumerClient(
live_eventhub['hostname'], live_eventhub['event_hub'], '$default',
EventHubSharedKeyCredential(live_eventhub['key_name'],
live_eventhub['access_key']))
with client:
partition_ids = client.get_partition_ids()
assert partition_ids == ['0', '1']
def test_get_properties(live_eventhub):
client = EventHubConsumerClient(
live_eventhub['hostname'], live_eventhub['event_hub'], '$default',
EventHubSharedKeyCredential(live_eventhub['key_name'],
live_eventhub['access_key']))
properties = client.get_eventhub_properties()
assert properties['eventhub_name'] == live_eventhub[
'event_hub'] and properties['partition_ids'] == ['0', '1']
client.close()
def test_get_partition_properties(live_eventhub):
client = EventHubConsumerClient(live_eventhub['hostname'], live_eventhub['event_hub'],
EventHubSharedKeyCredential(live_eventhub['key_name'], live_eventhub['access_key']))
properties = client.get_partition_properties('0')
assert properties['event_hub_path'] == live_eventhub['event_hub'] \
and properties['id'] == '0' \
and 'beginning_sequence_number' in properties \
and 'last_enqueued_sequence_number' in properties \
and 'last_enqueued_offset' in properties \
and 'last_enqueued_time_utc' in properties \
and 'is_empty' in properties
client.close()
def consume_events(starting_position):
global consumer_group
global events_received_count
global max_events_to_receive
global client
event_count = 0
eh_conn_str = os.environ['AZURE_STREAMPOC_EVENTHUB_CONN_STRING']
eh_namespace = os.environ['AZURE_STREAMPOC_EVENTHUB_NAMESPACE']
eh_hubname = os.environ['AZURE_STREAMPOC_EVENTHUB_HUBNAME']
zipcodes = read_nc_zipcodes_data()
start_epoch = arrow.utcnow().timestamp
consumer_group = reformat_consumer_group(consumer_group)
print('consume_events')
print('consumer_group: {}'.format(consumer_group))
print('starting_position: {}'.format(starting_position))
print('max_events_to_receive: {}'.format(max_events_to_receive))
print('eventhub namespace: {}'.format(eh_namespace))
print('eventhub hubname: {}'.format(eh_hubname))
print('eventhub conn_str: {}'.format(eh_conn_str))
event_count = 0
if max_events_to_receive > 0:
print('creating EventHubConsumerClient')
client = EventHubConsumerClient.from_connection_string(
eh_conn_str, consumer_group, eventhub_name=eh_hubname)
# "-1" is from the beginning of the partition.
client.receive(on_event=on_event_received, starting_position=starting_position)
def test_client_secret_credential(aad_credential, live_eventhub):
try:
from azure.identity import EnvironmentCredential
except ImportError:
pytest.skip("No azure identity library")
credential = EnvironmentCredential()
producer_client = EventHubProducerClient(
host=live_eventhub['hostname'],
event_hub_path=live_eventhub['event_hub'],
credential=credential,
user_agent='customized information')
consumer_client = EventHubConsumerClient(
host=live_eventhub['hostname'],
event_hub_path=live_eventhub['event_hub'],
credential=credential,
user_agent='customized information')
with producer_client:
producer_client.send(EventData(body='A single message'))
def on_events(partition_context, events):
assert partition_context.partition_id == '0'
assert len(events) == 1
assert list(events[0].body)[0] == 'A single message'.encode('utf-8')
with consumer_client:
worker = threading.Thread(target=consumer_client.receive,
args=(on_events, ),
kwargs={
"consumer_group": '$default',
"partition_id": '0'
})
worker.start()
time.sleep(2)
def test_receive_end_of_stream(connstr_senders):
def on_event(partition_context, event):
if partition_context.partition_id == "0":
assert event.body_as_str() == "Receiving only a single event"
assert list(event.body)[0] == b"Receiving only a single event"
on_event.called = True
on_event.called = False
connection_str, senders = connstr_senders
client = EventHubConsumerClient.from_connection_string(connection_str)
with client:
thread = threading.Thread(target=client.receive,
args=(on_event, "$default"),
kwargs={
"partition_id": "0",
"initial_event_position": "@latest"
})
thread.daemon = True
thread.start()
time.sleep(10)
assert on_event.called is False
senders[0].send(EventData(b"Receiving only a single event"))
time.sleep(10)
assert on_event.called is True
thread.join()
def test_receive_over_websocket_sync(connstr_senders):
app_prop = {"raw_prop": "raw_value"}
def on_event(partition_context, event):
on_event.received.append(event)
on_event.app_prop = event.application_properties
on_event.received = []
on_event.app_prop = None
connection_str, senders = connstr_senders
client = EventHubConsumerClient.from_connection_string(
connection_str, transport_type=TransportType.AmqpOverWebsocket)
event_list = []
for i in range(5):
ed = EventData("Event Number {}".format(i))
ed.application_properties = app_prop
event_list.append(ed)
senders[0].send(event_list)
with client:
thread = threading.Thread(target=client.receive,
args=(on_event, "$default"),
kwargs={
"partition_id": "0",
"initial_event_position": "-1"
})
thread.start()
time.sleep(10)
assert len(on_event.received) == 5
for ed in on_event.received:
assert ed.application_properties[b"raw_prop"] == b"raw_value"
def __init__(self, arguments):
super().__init__(arguments)
connection_string = self.get_from_env("AZURE_EVENTHUB_CONNECTION_STRING")
eventhub_name = self.get_from_env("AZURE_EVENTHUB_NAME")
self.async_producer = AsyncEventHubProducerClient.from_connection_string(connection_string, eventhub_name=eventhub_name)
self.consumer = EventHubConsumerClient.from_connection_string(connection_string, _EventHubTest.consumer_group, eventhub_name=eventhub_name)
self.async_consumer = AsyncEventHubConsumerClient.from_connection_string(connection_string, _EventHubTest.consumer_group, eventhub_name=eventhub_name)
def example_eventhub_consumer_ops():
# [START eventhub_consumer_client_close_sync]
import os
import threading
event_hub_connection_str = os.environ['EVENT_HUB_CONN_STR']
eventhub_name = os.environ['EVENT_HUB_NAME']
from azure.eventhub import EventHubConsumerClient
consumer = EventHubConsumerClient.from_connection_string(
conn_str=event_hub_connection_str,
consumer_group="$Default",
eventhub_name=eventhub_name)
logger = logging.getLogger("azure.eventhub")
def on_event(partition_context, event):
logger.info("Received event from partition: {}".format(
partition_context.partition_id))
# Do ops on the received event
# The 'receive' method is a blocking call, it can be executed in a thread for
# non-blocking behavior, and combined with the 'close' method.
worker = threading.Thread(target=consumer.receive,
kwargs={"on_event": on_event})
worker.start()
time.sleep(10) # Keep receiving for 10s then close.
# Close down the consumer handler explicitly.
consumer.close()
def __init__(self, opts):
self._opts = opts
self._client = None
self._mode = None
self._state = None
self._verbose = False
conn_str = opts['conn_str']
hub_name = opts['hub_name']
if 'verbose' in opts:
if opts['verbose'] == True:
self._verbose = True
if self.is_consumer():
consumer_group = opts['consumer_group']
self._client = EventHubConsumerClient.from_connection_string(
conn_str, consumer_group, eventhub_name=hub_name)
self._mode = 'consumer'
self._state = 'open'
else:
self._client = EventHubProducerClient.from_connection_string(
conn_str, eventhub_name=hub_name)
self._mode = 'producer'
self._state = 'open'
if self._verbose:
print("EventHub.__init__ client: {}".format(str(type(
self._client))))
def test_receive_end_of_stream(connstr_senders):
def on_event(partition_context, event):
if partition_context.partition_id == "0":
assert event.body_as_str() == "Receiving only a single event"
assert list(event.body)[0] == b"Receiving only a single event"
on_event.called = True
assert event.partition_key == b'0'
event_str = str(event)
assert ", offset: " in event_str
assert ", sequence_number: " in event_str
assert ", enqueued_time: " in event_str
assert ", partition_key: 0" in event_str
on_event.called = False
connection_str, senders = connstr_senders
client = EventHubConsumerClient.from_connection_string(
connection_str, consumer_group='$default')
with client:
thread = threading.Thread(target=client.receive,
args=(on_event, ),
kwargs={
"partition_id": "0",
"starting_position": "@latest"
})
thread.daemon = True
thread.start()
time.sleep(10)
assert on_event.called is False
senders[0].send(EventData(b"Receiving only a single event"),
partition_key='0')
time.sleep(10)
assert on_event.called is True
thread.join()
def test_receive_no_partition(connstr_senders):
connection_str, senders = connstr_senders
senders[0].send(EventData("Test EventData"))
senders[1].send(EventData("Test EventData"))
client = EventHubConsumerClient.from_connection_string(connection_str, consumer_group='$default', receive_timeout=1)
def on_event(partition_context, event):
on_event.received += 1
partition_context.update_checkpoint(event)
on_event.namespace = partition_context.fully_qualified_namespace
on_event.eventhub_name = partition_context.eventhub_name
on_event.consumer_group = partition_context.consumer_group
on_event.offset = event.offset
on_event.sequence_number = event.sequence_number
on_event.received = 0
on_event.namespace = None
on_event.eventhub_name = None
on_event.consumer_group = None
on_event.offset = None
on_event.sequence_number = None
with client:
worker = threading.Thread(target=client.receive,
args=(on_event,),
kwargs={"starting_position": "-1"})
worker.start()
time.sleep(10)
assert on_event.received == 2
checkpoints = list(client._event_processors.values())[0]._checkpoint_store.list_checkpoints(
on_event.namespace, on_event.eventhub_name, on_event.consumer_group
)
assert len([checkpoint for checkpoint in checkpoints if checkpoint["offset"] == on_event.offset]) > 0
assert len([checkpoint for checkpoint in checkpoints if checkpoint["sequence_number"] == on_event.sequence_number]) > 0
def run(self, config: IotHubConfig) -> None:
"""
Process queue and upload to CDF
Args:
config: Configuration parameters
"""
CONNECTION_STR = f"Endpoint={config.azureiothub.eventhub_compatible_endpoint}/;SharedAccessKeyName=service;SharedAccessKey={config.azureiothub.iot_sas_key};EntityPath={config.azureiothub.eventhub_compatible_path}"
client = EventHubConsumerClient.from_connection_string(
conn_str=CONNECTION_STR,
consumer_group="$default",
# transport_type=TransportType.AmqpOverWebsocket, # uncomment it if you want to use web socket
# http_proxy={ # uncomment if you want to use proxy
# 'proxy_hostname': '127.0.0.1', # proxy hostname.
# 'proxy_port': 3128, # proxy port.
# 'username': '******',
# 'password': '******'
# }
)
self.asset_id = cdf_client.assets.retrieve(
external_id=config.azureiothub.iot_root).id
try:
with client:
client.receive_batch(
on_event_batch=self.on_event_batch,
on_error=self.on_error,
)
except KeyboardInterrupt:
print("Receiving has stopped.")
def example_eventhub_consumer_ops():
# [START eventhub_consumer_client_close_sync]
import os
import threading
event_hub_connection_str = os.environ['EVENT_HUB_CONN_STR']
event_hub = os.environ['EVENT_HUB_NAME']
from azure.eventhub import EventHubConsumerClient
consumer = EventHubConsumerClient.from_connection_string(
conn_str=event_hub_connection_str, event_hub_path=event_hub)
logger = logging.getLogger("azure.eventhub")
def on_events(partition_context, events):
logger.info("Received {} messages from partition: {}".format(
len(events), partition_context.partition_id))
# Do ops on received events
# The receive method is blocking call, so execute it in a thread to
# better demonstrate how to stop the receiving by calling he close method.
worker = threading.Thread(target=consumer.receive,
kwargs={
"on_events": on_events,
"consumer_group": "$Default"
})
worker.start()
time.sleep(10) # Keep receiving for 10s then close.
# Close down the consumer handler explicitly.
consumer.close()
def test_receive_connection_idle_timeout_and_reconnect_sync(connstr_senders):
connection_str, senders = connstr_senders
client = EventHubConsumerClient.from_connection_string(
conn_str=connection_str, consumer_group='$default', idle_timeout=10)
def on_event_received(event):
on_event_received.event = event
with client:
consumer = client._create_consumer("$default", "0", "-1",
on_event_received)
with consumer:
consumer._open()
time.sleep(11)
ed = EventData("Event")
senders[0].send(ed)
consumer._handler.do_work()
assert consumer._handler._connection._state == c_uamqp.ConnectionState.DISCARDING
duration = 10
now_time = time.time()
end_time = now_time + duration
while now_time < end_time:
consumer.receive()
time.sleep(0.01)
now_time = time.time()
assert on_event_received.event.body_as_str() == "Event"
def test_receive_partition(connstr_senders):
connection_str, senders = connstr_senders
senders[0].send(EventData("Test EventData"))
client = EventHubConsumerClient.from_connection_string(
connection_str, consumer_group='$default')
def on_event(partition_context, event):
on_event.received += 1
on_event.partition_id = partition_context.partition_id
on_event.consumer_group = partition_context.consumer_group
on_event.fully_qualified_namespace = partition_context.fully_qualified_namespace
on_event.eventhub_name = partition_context.eventhub_name
on_event.received = 0
with client:
worker = threading.Thread(target=client.receive,
args=(on_event, ),
kwargs={
"starting_position": "-1",
"partition_id": "0"
})
worker.start()
time.sleep(10)
assert on_event.received == 1
assert on_event.partition_id == "0"
assert on_event.consumer_group == "$default"
assert on_event.fully_qualified_namespace in connection_str
assert on_event.eventhub_name == senders[0]._client.eventhub_name
def test_receive_batch_early_callback(connstr_senders):
''' Test whether the callback is called once max_batch_size reaches and before max_wait_time reaches.
'''
connection_str, senders = connstr_senders
for _ in range(10):
senders[0].send(EventData("Test EventData"))
client = EventHubConsumerClient.from_connection_string(
connection_str, consumer_group='$default')
def on_event_batch(partition_context, event_batch):
on_event_batch.received += len(event_batch)
on_event_batch.received = 0
with client:
worker = threading.Thread(target=client.receive_batch,
args=(on_event_batch, ),
kwargs={
"max_batch_size": 10,
"max_wait_time": 100,
"starting_position": "-1",
"partition_id": "0"
})
worker.start()
time.sleep(10)
assert on_event_batch.received == 10
worker.join()
def event_hub_task():
# Create the event hub client to receive messages from IoT hub
client = EventHubConsumerClient.from_connection_string(
conn_str=connection_string, consumer_group=consumer_group_name)
# Set up the batch receiving of messages
with client:
client.receive_batch(on_event_batch=on_event_batch)
def send(self):
client = EventHubConsumerClient.from_connection_string(
conn_str=self.connectionString,
consumer_group="$default",
)
try:
with client:
client.receive_batch(on_event_batch=self.on_event_batch,
on_error=self.on_error)
except KeyboardInterrupt:
print("Primanje je završilo.")
def __init__(self):
# This test requires a previusly created Event Hub.
# In this example the name is "myeventhub", but it could be change below
connection_string = os.environ["EVENT_HUBS_CONNECTION_STRING"]
event_hub_name = "myeventhub"
self.consumer_client = EventHubConsumerClient.from_connection_string(
connection_string, CONSUMER_GROUP, idle_timeout=RECEIVE_TIMEOUT)
self.producer_client = EventHubProducerClient.from_connection_string(
connection_string)
self.received_event_count = 0
def main():
client = EventHubConsumerClient.from_connection_string(
conn_str=CONNECTION_STR_SUB,
consumer_group="$default",
)
try:
with client:
client.receive_batch(on_event_batch=on_event_batch,
on_error=on_error)
except KeyboardInterrupt:
print("Receiving has stopped.")
def createClients(self):
try:
client = EventHubConsumerClient.from_connection_string(
conn_str=self.CONNECTION_STR_LSM,
consumer_group="$default",
)
clientJob = ClientJob(client, "LSM")
clientJob.start()
except KeyboardInterrupt:
print("Primanje je završilo.")
def test_receive_with_invalid_hostname_sync(invalid_hostname):
def on_event(partition_context, event):
pass
client = EventHubConsumerClient.from_connection_string(
invalid_hostname, consumer_group='$default')
with client:
thread = threading.Thread(target=client.receive, args=(on_event, ))
thread.start()
time.sleep(2)
assert len(client._event_processors) == 1
thread.join()
def test_custom_certificate():
producer = EventHubProducerClient("fake.host.com",
"fake_eh",
None,
connection_verify='/usr/bin/local/cert')
assert producer._config.connection_verify == '/usr/bin/local/cert'
consumer = EventHubConsumerClient("fake.host.com",
"fake_eh",
"fake_group",
None,
connection_verify='D:/local/certfile')
assert consumer._config.connection_verify == 'D:/local/certfile'
def test_get_properties_with_auth_error_sync(live_eventhub):
client = EventHubConsumerClient(
live_eventhub['hostname'], live_eventhub['event_hub'], '$default',
EventHubSharedKeyCredential(live_eventhub['key_name'],
"AaBbCcDdEeFf="))
with client:
with pytest.raises(AuthenticationError) as e:
client.get_eventhub_properties()
client = EventHubConsumerClient(
live_eventhub['hostname'], live_eventhub['event_hub'], '$default',
EventHubSharedKeyCredential("invalid", live_eventhub['access_key']))
with client:
with pytest.raises(AuthenticationError) as e:
client.get_eventhub_properties()
