def test_example_eventhub_sync_sender_ops(live_eventhub_config, connection_str): import os # [START create_eventhub_client_sender_instance] from azure.eventhub import EventHubClient client = EventHubClient.from_connection_string(connection_str) sender = client.add_sender(partition="0") # [END create_eventhub_client_sender_instance] # [START eventhub_client_sender_open] client = EventHubClient.from_connection_string(connection_str) sender = client.add_sender(partition="0") try: # Open the Sender using the supplied conneciton. sender.open() # Start sending except: raise finally: # Close down the send handler. sender.close() # [END eventhub_client_sender_open] # [START eventhub_client_sender_close] client = EventHubClient.from_connection_string(connection_str) sender = client.add_sender(partition="0") try: # Open the Sender using the supplied conneciton. sender.open() # Start sending except: raise finally: # Close down the send handler. sender.close()
def add_receiver(client: EventHubClient, offset: Offset): receiver = client.add_receiver(CONSUMER_GROUP, EVENT_HUB_PARTITION, prefetch=PREFETCH, offset=offset) client.run() return receiver
class Consumer: consumer_group = None eventhubs_client = None offset = Offset("-1") redis_cache = None def __init__(self, eventhub, address, user, key, consumer_group, redis_hostname, redis_key): self.consumer_group = consumer_group self.eventhubs_client = EventHubClient(address, debug=False, username=user, password=key) redis_topic = f"eventhubs-{eventhub}-{consumer_group}" self.redis_cache = RedisCache(redis_hostname, redis_key, redis_topic) def recieve(self): OFFSET = Offset(self.redis_cache.get_offset()) receiver = self.eventhubs_client.add_receiver(self.consumer_group, "0", prefetch=5000, offset=OFFSET) self.eventhubs_client.run() messages = receiver.receive(timeout=100) self.eventhubs_client.stop() return messages def commit(self, event_data): self.redis_cache.set_offset(event_data.sequence_number)
def test_client_secret_credential(aad_credential, live_eventhub): try: from azure.identity import ClientSecretCredential except ImportError: pytest.skip("No azure identity library") client_id, secret, tenant_id = aad_credential credential = ClientSecretCredential(client_id=client_id, secret=secret, tenant_id=tenant_id) client = EventHubClient(host=live_eventhub['hostname'], event_hub_path=live_eventhub['event_hub'], credential=credential, user_agent='customized information') sender = client.create_producer(partition_id='0') receiver = client.create_consumer(consumer_group="$default", partition_id='0', event_position=EventPosition("@latest")) with receiver: received = receiver.receive(timeout=1) assert len(received) == 0 with sender: event = EventData(body='A single message') sender.send(event) time.sleep(1) received = receiver.receive(timeout=1) assert len(received) == 1 assert list(received[0].body)[0] == 'A single message'.encode('utf-8')
def main(req: func.HttpRequest) -> func.HttpResponse: logging.info('Python HTTP trigger function processed a request.') connection_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format( os.environ['EVENT_HUB_HOSTNAME'], os.environ['EVENT_HUB_SAS_POLICY'], os.environ['EVENT_HUB_SAS_KEY'], os.environ['EVENT_HUB_NAME']) client = EventHubClient.from_connection_string(connection_str) req_body = req.get_json() req_payload = json.dumps(req_body) client = EventHubClient.from_connection_string(connection_str) sender = client.add_sender(partition="0") try: client.run() logging.info('Send Alert with this payload: %s', req_payload) event_data = EventData(req_payload) sender.send(event_data) logging.info('Sent payload to Event Hub!') except: raise finally: client.stop() return func.HttpResponse(f"Hello {req_body}!")
def get(): if request.args.get('since') is None: since = -1 else: since = request.args.get('since') client = EventHubClient(address, debug=False, username=user, password=key) receiver = client.add_receiver(consumergroup, PARTITION, prefetch=1000, offset=Offset(since), keep_alive=72000) client.run() def generate(): batched_events = receiver.receive(max_batch_size=100, timeout=500) yield '[' index = 0 while batched_events: for event_data in batched_events: if index > 0: yield ',' last_sn = event_data.sequence_number data = str(event_data.message) output_entity = literal_eval(data) output_entity.update({"_updated": str(last_sn)}) yield json.dumps(output_entity) index = index + 1 batched_events = receiver.receive(max_batch_size=100, timeout=500) yield ']' return Response(generate(), mimetype='application/json')
def __init__(self, address, user, key): self.client = EventHubClient(address, debug=False, username=user, password=key) self.sender = self.client.add_sender(partition="0") self.client.run()
def __init__(self, config): """ Class to create an EventHubStreamingClient instance. :param config: Dictionary file with all the relevant parameters. """ super().__init__() self.message_callback = None self.config = config self.storage_account_name = self.config.get("AZURE_STORAGE_ACCOUNT") self.storage_key = self.config.get("AZURE_STORAGE_ACCESS_KEY") self.lease_container_name = self.config.get("LEASE_CONTAINER_NAME") self.namespace = self.config.get("EVENT_HUB_NAMESPACE") self.eventhub = self.config.get("EVENT_HUB_NAME") self.consumer_group = self.config.get("EVENT_HUB_CONSUMER_GROUP") if self.consumer_group is None: self.consumer_group = '$default' self.user = self.config.get("EVENT_HUB_SAS_POLICY") self.key = self.config.get("EVENT_HUB_SAS_KEY") if self.config.get("TIMEOUT"): try: self.timeout = int(self.config.get("TIMEOUT")) except ValueError: self.timeout = None else: self.timeout = None # Create EPH Client if self.storage_account_name is not None and self.storage_key is not None: self.eph_client = EventHubConfig( sb_name=self.namespace, eh_name=self.eventhub, policy=self.user, sas_key=self.key, consumer_group=self.consumer_group) self.eh_options = EPHOptions() self.eh_options.release_pump_on_timeout = True self.eh_options.auto_reconnect_on_error = False self.eh_options.debug_trace = False self.storage_manager = AzureStorageCheckpointLeaseManager( self.storage_account_name, self.storage_key, self.lease_container_name) # Create Send client else: address = "amqps://" + self.namespace + \ ".servicebus.windows.net/" + self.eventhub try: self.send_client = EventHubClient(address, debug=False, username=self.user, password=self.key) self.sender = self.send_client.add_sender() self.send_client.run() except Exception as e: logger.error('Failed to init EH send client: ' + str(e)) raise
def test_long_running_receive(connection_str): parser = argparse.ArgumentParser() parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30) parser.add_argument("--consumer", help="Consumer group name", default="$default") parser.add_argument("--partitions", help="Comma seperated partition IDs") parser.add_argument("--offset", help="Starting offset", default="-1") parser.add_argument("--conn-str", help="EventHub connection string", default=connection_str) parser.add_argument("--eventhub", help="Name of EventHub") parser.add_argument("--address", help="Address URI to the EventHub entity") parser.add_argument( "--sas-policy", help="Name of the shared access policy to authenticate with") parser.add_argument("--sas-key", help="Shared access key") args, _ = parser.parse_known_args() if args.conn_str: client = EventHubClient.from_connection_string( args.conn_str, event_hub_path=args.eventhub, network_tracing=False) elif args.address: client = EventHubClient(host=args.address, event_hub_path=args.eventhub, credential=EventHubSharedKeyCredential( args.sas_policy, args.sas_key), auth_timeout=240, network_tracing=False) else: try: import pytest pytest.skip("Must specify either '--conn-str' or '--address'") except ImportError: raise ValueError("Must specify either '--conn-str' or '--address'") if args.partitions: partitions = args.partitions.split(",") else: partitions = client.get_partition_ids() threads = [] for pid in partitions: consumer = client.create_consumer(consumer_group="$default", partition_id=pid, event_position=EventPosition( args.offset), prefetch=300) thread = threading.Thread(target=pump, args=(consumer, args.duration)) thread.start() threads.append(thread) for thread in threads: thread.join()
def __init__(self, eventhub, address, user, key, consumer_group, redis_hostname, redis_key): self.consumer_group = consumer_group self.eventhubs_client = EventHubClient(address, debug=False, username=user, password=key) redis_topic = f"eventhubs-{eventhub}-{consumer_group}" self.redis_cache = RedisCache(redis_hostname, redis_key, redis_topic)
def cosmosDBServiceToCosmosDB(self): database_link = 'dbs/' + DATABASE_ID collection_link = database_link + '/colls/' + COLLECTION_ID counter = 0 filepath = '' CONSUMER_GROUP = "$Default" OFFSET = Offset("0") PARTITION = "0" eh_client = EventHubClient('amqps://xxxxx.servicebus.windows.net/txxxxqueue', debug=True, username='******', password='******') receiver = eh_client.add_receiver(CONSUMER_GROUP, PARTITION, prefetch=300, offset=OFFSET) try: eh_client.run() while True: for event_data in receiver.receive(timeout=100): rcv_msg = str(event_data.message) # Filter the Null messages if len(rcv_msg)>5: # Load the messages in CosmosDB cosmos_client.CreateDocument(collection_link, json.loads(str(event_data.message))) eh_client.stop() except Exception as e: print("Failed Receiving Record {}".format(str(e)) ) finally: eh_client.stop()
def __init__(self, address, user, key): self.address = address self.user = user self.key = key self.counter = 0 self.client_batch = EventHubClient(self.address, debug=False, username=self.user, password=self.key) self.sender = self.client_batch.add_sender() self.client_batch.run()
def main(req: func.HttpRequest) -> func.HttpResponse: logger = logging.getLogger(__name__) formatter = logging.Formatter( '%(asctime)s %(name)s %(levelname)s: %(message)s') func_context = os.environ['FUNCTION_CONTEXT'] logger.debug(f"Function context --> {func_context}") credentials = None subscription_id = None kv_credentials = None kv_subscription_id = None if func_context == 'local': filehandler = logging.FileHandler('func.log') filehandler.setFormatter(formatter) logger.addHandler(filehandler) logger.setLevel(logging.DEBUG) credentials, subscription_id = get_local_credentials() else: console = logging.StreamHandler() console.setLevel(logging.INFO) console.setFormatter(formatter) credentials, subscription_id = get_azure_credentials() logger.debug('Python HTTP trigger function processed a request.') logger.debug(f"method={req.method}, url={req.url}, params={req.params}") logger.debug(f"body={req.get_json()}") # Handle WebHook webhook = req.get_json() #Create an empty dict within webhook for motsID webhook['additionalData'] = {} addl_data = webhook['additionalData'] addl_data['motsID'] = "0000" # Key Vault stuff. kv_mgmt_client = KeyVaultManagementClient(credentials, subscription_id) kv_client = KeyVaultClient(credentials) namespace = get_kv_secret(kv_client, 'EventHubNamespace') event_hub = get_kv_secret(kv_client, 'EventHub') user = get_kv_secret(kv_client, 'EventHubKeyName') key = get_kv_secret(kv_client, 'EventHubKey') amqp_uri = f"https://{namespace}.servicebus.windows.net/{event_hub}" eh_client = EventHubClient(amqp_uri, debug=False, username=user, password=key) eh_sender = eh_client.add_sender(partition="0") eh_client.run() eh_sender.send(EventData(json.dumps(webhook))) logger.info(f"sending event to {amqp_uri}, {json.dumps(webhook)}") date = datetime.datetime.now() return func.HttpResponse(json.dumps({'date': date, 'status': 'SUCCESS'}))
def __init__(self): #load config with open('config.json', 'r') as json_file: self.config = json.load(json_file) # Create Event Hubs client client = EventHubClient(self.config["EH_ADDRESS"], debug=False, username=self.config["EH_USER"], password=self.config["EH_KEY"]) Worker.event_hub_sender = client.add_sender(partition="0") client.run()
def init_event_hub(): with open('config.json', 'r') as json_file: config = json.load(json_file) client = EventHubClient(host=config["EH_HOST"], event_hub_path=config["EH_NAME"], credential=EventHubSharedKeyCredential( config["EVENT_HUB_SAS_POLICY"], config["EVENT_HUB_SAS_KEY"]), network_tracing=False) for i in range(NUM_PARTITIONS): event_producer_list.append(client.create_producer(partition_id=str(i)))
def test_get_partition_properties(live_eventhub): client = EventHubClient(live_eventhub['hostname'], live_eventhub['event_hub'], EventHubSharedKeyCredential(live_eventhub['key_name'], live_eventhub['access_key']) ) properties = client.get_partition_properties('0') assert properties['event_hub_path'] == live_eventhub['event_hub'] \ and properties['id'] == '0' \ and 'beginning_sequence_number' in properties \ and 'last_enqueued_sequence_number' in properties \ and 'last_enqueued_offset' in properties \ and 'last_enqueued_time_utc' in properties \ and 'is_empty' in properties
def test_long_running_send(connection_str): if sys.platform.startswith('darwin'): import pytest pytest.skip("Skipping on OSX") parser = argparse.ArgumentParser() parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30) parser.add_argument("--payload", help="payload size", type=int, default=512) parser.add_argument("--conn-str", help="EventHub connection string", default=connection_str) parser.add_argument("--eventhub", help="Name of EventHub") parser.add_argument("--address", help="Address URI to the EventHub entity") parser.add_argument( "--sas-policy", help="Name of the shared access policy to authenticate with") parser.add_argument("--sas-key", help="Shared access key") args, _ = parser.parse_known_args() if args.conn_str: client = EventHubClient.from_connection_string( args.conn_str, event_hub_path=args.eventhub) elif args.address: client = EventHubClient(host=args.address, event_hub_path=args.eventhub, credential=EventHubSharedKeyCredential( args.sas_policy, args.sas_key), auth_timeout=240, network_tracing=False) else: try: import pytest pytest.skip("Must specify either '--conn-str' or '--address'") except ImportError: raise ValueError("Must specify either '--conn-str' or '--address'") try: partition_ids = client.get_partition_ids() threads = [] for pid in partition_ids: sender = client.create_producer(partition_id=pid) thread = threading.Thread(target=send, args=(sender, args)) thread.start() threads.append(thread) thread.join() except KeyboardInterrupt: pass
def __init__(self, devenv='doh'): """ Create Event Hub Sender """ if devenv == 'prod': ADDRESS = "amqps://doh-airqual-eventhub.servicebus.windows.net/doh-airqual-event-hub-prod" USER = "******" KEY = "1p1aHCJc5IbamvnzlnvUa2wlvXsaJpSAbAORGlPRaQ4=" elif devenv == "dev": ADDRESS = "amqps://az-doh-airqual-eventhub.servicebus.windows.net/az-doh-airqual-eventhub" USER = "******" KEY = "pfWGtB6obtiUCwwAobAAuaq7B9SZSsnpHY6ArEbeS1A=" self.client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) self.sender = self.client.add_sender(partition="1") self.client.run()
def __init__(self): NAMESPACE = os.environ['EVENT_HUB_NAMESPACE'] EHNAME = os.environ['EVENT_HUB_NAME'] ADDRESS = "amqps://" + NAMESPACE + ".servicebus.windows.net/" + EHNAME # SAS policy and key are not required if they are encoded in the URL USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') self.client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) self.sender = self.client.add_sender(partition="0") self.client.run()
def test_send_with_partition_key(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubClient.from_connection_string(connection_str, debug=False) sender = client.add_sender() try: client.run() data_val = 0 for partition in [b"a", b"b", b"c", b"d", b"e", b"f"]: partition_key = b"test_partition_" + partition for i in range(50): data = EventData(str(data_val)) data.partition_key = partition_key data_val += 1 sender.send(data) except: raise finally: client.stop() found_partition_keys = {} for index, partition in enumerate(receivers): received = partition.receive(timeout=5) for message in received: try: existing = found_partition_keys[message.partition_key] assert existing == index except KeyError: found_partition_keys[message.partition_key] = index
def test_long_running_send(connection_str): if sys.platform.startswith('darwin'): import pytest pytest.skip("Skipping on OSX") parser = argparse.ArgumentParser() parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30) parser.add_argument("--payload", help="payload size", type=int, default=512) parser.add_argument("--batch", help="Number of events to send and wait", type=int, default=1) parser.add_argument("--conn-str", help="EventHub connection string", default=connection_str) parser.add_argument("--eventhub", help="Name of EventHub") parser.add_argument("--address", help="Address URI to the EventHub entity") parser.add_argument("--sas-policy", help="Name of the shared access policy to authenticate with") parser.add_argument("--sas-key", help="Shared access key") args, _ = parser.parse_known_args() if args.conn_str: client = EventHubClient.from_connection_string( args.conn_str, eventhub=args.eventhub) elif args.address: client = EventHubClient( args.address, username=args.sas_policy, password=args.sas_key) else: try: import pytest pytest.skip("Must specify either '--conn-str' or '--address'") except ImportError: raise ValueError("Must specify either '--conn-str' or '--address'") try: main(client, args) except KeyboardInterrupt: pass
def create_eventhub_client_from_iothub_connection_string(live_eventhub_config): # [START create_eventhub_client_iot_connstr] import os from azure.eventhub import EventHubClient iot_connection_str = os.environ['IOTHUB_CONNECTION_STR'] client = EventHubClient.from_connection_string(iot_connection_str)
def test_receive_with_custom_datetime_sync(connstr_senders): connection_str, senders = connstr_senders client = EventHubClient.from_connection_string(connection_str, debug=False) for i in range(5): senders[0].send(EventData(b"Message before timestamp")) time.sleep(60) now = datetime.datetime.utcnow() offset = datetime.datetime(now.year, now.month, now.day, now.hour, now.minute) for i in range(5): senders[0].send(EventData(b"Message after timestamp")) receiver = client.add_receiver("$default", "0", offset=Offset(offset)) try: client.run() all_received = [] received = receiver.receive(timeout=1) while received: all_received.extend(received) received = receiver.receive(timeout=1) assert len(all_received) == 5 for received_event in all_received: assert received_event.body_as_str() == "Message after timestamp" assert received_event.enqueued_time > offset except: raise finally: client.stop()
def test_receive_with_inclusive_offset(connstr_senders): connection_str, senders = connstr_senders client = EventHubClient.from_connection_string(connection_str, network_tracing=False) receiver = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition('@latest')) with receiver: received = receiver.receive(timeout=5) assert len(received) == 0 senders[0].send(EventData(b"Data")) time.sleep(1) received = receiver.receive(timeout=5) assert len(received) == 1 offset = received[0].offset assert list(received[0].body) == [b'Data'] assert received[0].body_as_str() == "Data" offset_receiver = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition( offset, inclusive=True)) with offset_receiver: received = offset_receiver.receive(timeout=5) assert len(received) == 1
def test_receive_with_offset_sync(connstr_senders): connection_str, senders = connstr_senders client = EventHubClient.from_connection_string(connection_str, debug=False) partitions = client.get_eventhub_info() assert partitions["partition_ids"] == ["0", "1"] receiver = client.add_receiver("$default", "0", offset=Offset('@latest')) try: client.run() more_partitions = client.get_eventhub_info() assert more_partitions["partition_ids"] == ["0", "1"] received = receiver.receive(timeout=5) assert len(received) == 0 senders[0].send(EventData(b"Data")) received = receiver.receive(timeout=5) assert len(received) == 1 offset = received[0].offset assert list(received[0].body) == [b'Data'] assert received[0].body_as_str() == "Data" offset_receiver = client.add_receiver("$default", "0", offset=offset) client.run() received = offset_receiver.receive(timeout=5) assert len(received) == 0 senders[0].send(EventData(b"Message after offset")) received = offset_receiver.receive(timeout=5) assert len(received) == 1 except: raise finally: client.stop()
def test_receive_with_sequence_no(connstr_senders): connection_str, senders = connstr_senders client = EventHubClient.from_connection_string(connection_str, debug=False) receiver = client.add_receiver("$default", "0", offset=Offset('@latest')) try: client.run() received = receiver.receive(timeout=5) assert len(received) == 0 senders[0].send(EventData(b"Data")) time.sleep(1) received = receiver.receive(timeout=5) assert len(received) == 1 offset = received[0].sequence_number offset_receiver = client.add_receiver("$default", "0", offset=Offset(offset)) client.run() received = offset_receiver.receive(timeout=5) assert len(received) == 0 senders[0].send(EventData(b"Message next in sequence")) time.sleep(1) received = offset_receiver.receive(timeout=5) assert len(received) == 1 except: raise finally: client.stop()
def test_send_with_partition_key(connection_str, receivers): client = EventHubClient.from_connection_string(connection_str, debug=False) sender = client.add_sender() try: client.run() data_val = 0 for partition in [b"a", b"b", b"c", b"d", b"e", b"f"]: partition_key = b"test_partition_" + partition for i in range(50): data = EventData(str(data_val)) data.partition_key = partition_key data_val += 1 sender.send(data) except: raise finally: client.stop() found_partition_keys = {} for index, partition in enumerate(receivers): received = partition.receive(timeout=5) for message in received: try: existing = found_partition_keys[message.partition_key] assert existing == index except KeyError: found_partition_keys[message.partition_key] = index
def test_receive_with_inclusive_offset(connstr_senders): connection_str, senders = connstr_senders client = EventHubClient.from_connection_string(connection_str, debug=False) receiver = client.add_receiver("$default", "0", offset=Offset('@latest')) try: client.run() received = receiver.receive(timeout=5) assert len(received) == 0 senders[0].send(EventData(b"Data")) time.sleep(1) received = receiver.receive(timeout=5) assert len(received) == 1 offset = received[0].offset assert list(received[0].body) == [b'Data'] assert received[0].body_as_str() == "Data" offset_receiver = client.add_receiver("$default", "0", offset=Offset(offset.value, inclusive=True)) client.run() received = offset_receiver.receive(timeout=5) assert len(received) == 1 except: raise finally: client.stop()
def test_send_with_forced_conn_close_sync(connstr_receivers, sleep): pytest.skip("This test is similar to the above one") connection_str, receivers = connstr_receivers client = EventHubClient.from_connection_string(connection_str, network_tracing=False) sender = client.create_producer() with sender: sender.send(EventData(b"A single event")) sender._handler._connection._conn.destroy() if sleep: time.sleep(300) else: sender._handler._connection._conn.destroy() sender.send(EventData(b"A single event")) sender.send(EventData(b"A single event")) if sleep: time.sleep(300) else: sender._handler._connection._conn.destroy() sender.send(EventData(b"A single event")) sender.send(EventData(b"A single event")) received = [] for r in receivers: if not sleep: r._handler._connection._conn.destroy() received.extend(r.receive(timeout=1)) assert len(received) == 5 assert list(received[0].body)[0] == b"A single event"
def create_eventhub_client_from_iothub_connection_string(live_eventhub_config): # [START create_eventhub_client_iot_connstr] import os from azure.eventhub import EventHubClient iot_connection_str = os.environ['IOTHUB_CONNECTION_STR'] client = EventHubClient.from_iothub_connection_string(iot_connection_str)
def __init__(self): # This test requires a previusly created Event Hub. # In this example the name is "myeventhub", but it could be change below connectionString = os.environ["EVENT_HUBS_CONNECTION_STRING"] eventHubName = "myeventhub" self.client = EventHubClient.from_connection_string( connectionString, eventHubName)
def __init__(self): connection_string = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format( Settings.eh.hostname, Settings.eh.sas_policy, Settings.eh.sas_key, Settings.eh.event_hub_name) self.client = EventHubClient.from_connection_string(connection_string) self.sender = self.client.add_sender(partition="0") self.client.run()
def test_send_batch_with_app_prop_sync(connstr_receivers): connection_str, receivers = connstr_receivers app_prop_key = "raw_prop" app_prop_value = "raw_value" app_prop = {app_prop_key: app_prop_value} def batched(): for i in range(10): ed = EventData("Event number {}".format(i)) ed.application_properties = app_prop yield ed for i in range(10, 20): ed = EventData("Event number {}".format(i)) ed.application_properties = app_prop yield ed client = EventHubClient.from_connection_string(connection_str, network_tracing=False) sender = client.create_producer() with sender: sender.send(batched()) time.sleep(1) received = [] for r in receivers: received.extend(r.receive(timeout=3)) assert len(received) == 20 for index, message in enumerate(received): assert list(message.body)[0] == "Event number {}".format(index).encode('utf-8') assert (app_prop_key.encode('utf-8') in message.application_properties) \ and (dict(message.application_properties)[app_prop_key.encode('utf-8')] == app_prop_value.encode('utf-8'))
def start_event_generation_sync_impl(cancellation_token): random.seed(int(time.time())) # use ticks as seed client = EventHubClient.from_connection_string(conn_str=EVENTHUB_CONNECTION_STRING, eventhub=EVENTHUB_NAME) sender = client.add_sender() client.run() while cancellation_token.is_set(): try: # Simulate sending data from 100 weather sensors devices_data = [] for i in range(0, 100): scale_factor = random.randrange(0,25) windturbine_measure = generate_turbine_measure("Python_Turbine_" + str(i), scale_factor) ev_data = serialize_windturbine_to_eventdata(windturbine_measure) devices_data.append(ev_data) sender.send(EventData(batch=[event for event in devices_data])) logger.info("100 events sent!") print(".", end='', flush=True) except Exception as e: logger.error(e) client.stop()
def test_send_to_invalid_partitions(connection_str): partitions = ["XYZ", "-1", "1000", "-" ] for p in partitions: client = EventHubClient.from_connection_string(connection_str, debug=False) sender = client.add_sender(partition=p) try: with pytest.raises(EventHubError): client.run() finally: client.stop()
def create_eventhub_client_from_sas_token(live_eventhub_config): # [START create_eventhub_client_sas_token] import os from azure.eventhub import EventHubClient address = os.environ['EVENT_HUB_ADDRESS'] sas_token = os.environ['EVENT_HUB_SAS_TOKEN'] client = EventHubClient.from_sas_token( address=address, sas_token=sas_token)
def test_receive_from_invalid_partitions_sync(connection_str): partitions = ["XYZ", "-1", "1000", "-" ] for p in partitions: client = EventHubClient.from_connection_string(connection_str, debug=True) receiver = client.add_receiver("$default", p) try: with pytest.raises(EventHubError): client.run() receiver.receive(timeout=10) finally: client.stop()
def test_iothub_receive_sync(iot_connection_str, device_id): client = EventHubClient.from_iothub_connection_string(iot_connection_str, debug=True) receiver = client.add_receiver("$default", "0", operation='/messages/events') try: client.run() partitions = client.get_eventhub_info() assert partitions["partition_ids"] == ["0", "1", "2", "3"] received = receiver.receive(timeout=5) assert len(received) == 0 finally: client.stop()
def test_send_null_body(connection_str): partitions = ["XYZ", "-1", "1000", "-" ] client = EventHubClient.from_connection_string(connection_str, debug=False) sender = client.add_sender() try: client.run() with pytest.raises(ValueError): data = EventData(None) sender.send(data) finally: client.stop()
def test_iothub_send_single_event(iot_connection_str, device_id): client = EventHubClient.from_iothub_connection_string(iot_connection_str, debug=True) sender = client.add_sender(operation='/messages/devicebound') try: client.run() outcome = sender.send(EventData(b"A single event", to_device=device_id)) assert outcome.value == 0 except: raise finally: client.stop()
def test_send_partition_key_with_partition_sync(connection_str): client = EventHubClient.from_connection_string(connection_str, debug=True) sender = client.add_sender(partition="1") try: client.run() data = EventData(b"Data") data.partition_key = b"PKey" with pytest.raises(ValueError): sender.send(data) finally: client.stop()
def connstr_senders(connection_str): client = EventHubClient.from_connection_string(connection_str, debug=True) eh_hub_info = client.get_eventhub_info() partitions = eh_hub_info["partition_ids"] senders = [] for p in partitions: senders.append(client.add_sender(partition=p)) client.run() yield connection_str, senders client.stop()
def test_send_too_large_message(connection_str): if sys.platform.startswith('darwin'): pytest.skip("Skipping on OSX - open issue regarding message size") client = EventHubClient.from_connection_string(connection_str, debug=True) sender = client.add_sender() try: client.run() data = EventData(b"A" * 300000) with pytest.raises(EventHubError): sender.send(data) finally: client.stop()
def test_message_body_types(connstr_senders): connection_str, senders = connstr_senders client = EventHubClient.from_connection_string(connection_str, debug=False) receiver = client.add_receiver("$default", "0", offset=Offset('@latest')) try: client.run() received = receiver.receive(timeout=5) assert len(received) == 0 senders[0].send(EventData(b"Bytes Data")) time.sleep(1) received = receiver.receive(timeout=5) assert len(received) == 1 assert list(received[0].body) == [b'Bytes Data'] assert received[0].body_as_str() == "Bytes Data" with pytest.raises(TypeError): received[0].body_as_json() senders[0].send(EventData("Str Data")) time.sleep(1) received = receiver.receive(timeout=5) assert len(received) == 1 assert list(received[0].body) == [b'Str Data'] assert received[0].body_as_str() == "Str Data" with pytest.raises(TypeError): received[0].body_as_json() senders[0].send(EventData(b'{"test_value": "JSON bytes data", "key1": true, "key2": 42}')) time.sleep(1) received = receiver.receive(timeout=5) assert len(received) == 1 assert list(received[0].body) == [b'{"test_value": "JSON bytes data", "key1": true, "key2": 42}'] assert received[0].body_as_str() == '{"test_value": "JSON bytes data", "key1": true, "key2": 42}' assert received[0].body_as_json() == {"test_value": "JSON bytes data", "key1": True, "key2": 42} senders[0].send(EventData('{"test_value": "JSON str data", "key1": true, "key2": 42}')) time.sleep(1) received = receiver.receive(timeout=5) assert len(received) == 1 assert list(received[0].body) == [b'{"test_value": "JSON str data", "key1": true, "key2": 42}'] assert received[0].body_as_str() == '{"test_value": "JSON str data", "key1": true, "key2": 42}' assert received[0].body_as_json() == {"test_value": "JSON str data", "key1": True, "key2": 42} senders[0].send(EventData(42)) time.sleep(1) received = receiver.receive(timeout=5) assert len(received) == 1 assert received[0].body_as_str() == "42" assert received[0].body == 42 except: raise finally: client.stop()
def test_example_eventhub_sync_receiver_ops(live_eventhub_config, connection_str): import os # [START create_eventhub_client_receiver_instance] from azure.eventhub import EventHubClient, Offset client = EventHubClient.from_connection_string(connection_str) receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) # [END create_eventhub_client_receiver_instance] # [START eventhub_client_receiver_open] client = EventHubClient.from_connection_string(connection_str) receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) try: # Open the Receiver using the supplied conneciton. receiver.open() # Start receiving except: raise finally: # Close down the receive handler. receiver.close() # [END eventhub_client_receiver_open] # [START eventhub_client_receiver_close] client = EventHubClient.from_connection_string(connection_str) receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) try: # Open the Receiver using the supplied conneciton. receiver.open() # Start receiving except: raise finally: # Close down the receive handler. receiver.close() # [END eventhub_client_receiver_close]
def test_send_partition(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubClient.from_connection_string(connection_str, debug=False) sender = client.add_sender(partition="1") try: client.run() sender.send(EventData(b"Data")) except: raise finally: client.stop() partition_0 = receivers[0].receive(timeout=2) assert len(partition_0) == 0 partition_1 = receivers[1].receive(timeout=2) assert len(partition_1) == 1
def connstr_receivers(connection_str): client = EventHubClient.from_connection_string(connection_str, debug=False) eh_hub_info = client.get_eventhub_info() partitions = eh_hub_info["partition_ids"] recv_offset = Offset("@latest") receivers = [] for p in partitions: receivers.append(client.add_receiver("$default", p, prefetch=500, offset=Offset("@latest"))) client.run() for r in receivers: r.receive(timeout=1) yield connection_str, receivers client.stop()
def test_send_non_ascii(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubClient.from_connection_string(connection_str, debug=False) sender = client.add_sender(partition="0") try: client.run() sender.send(EventData(u"é,è,à,ù,â,ê,î,ô,û")) sender.send(EventData(json.dumps({"foo": u"漢字"}))) except: raise finally: client.stop() partition_0 = receivers[0].receive(timeout=2) assert len(partition_0) == 2 assert partition_0[0].body_as_str() == u"é,è,à,ù,â,ê,î,ô,û" assert partition_0[1].body_as_json() == {"foo": u"漢字"}
def test_send_multiple_clients(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubClient.from_connection_string(connection_str, debug=False) sender_0 = client.add_sender(partition="0") sender_1 = client.add_sender(partition="1") try: client.run() sender_0.send(EventData(b"Message 0")) sender_1.send(EventData(b"Message 1")) except: raise finally: client.stop() partition_0 = receivers[0].receive(timeout=2) assert len(partition_0) == 1 partition_1 = receivers[1].receive(timeout=2) assert len(partition_1) == 1
def test_send_single_event(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubClient.from_connection_string(connection_str, debug=False) sender = client.add_sender() try: client.run() sender.send(EventData(b"A single event")) except: raise finally: client.stop() received = [] for r in receivers: received.extend(r.receive(timeout=1)) assert len(received) == 1 assert list(received[0].body)[0] == b"A single event"
def test_send_array_sync(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubClient.from_connection_string(connection_str, debug=True) sender = client.add_sender() try: client.run() sender.send(EventData([b"A", b"B", b"C"])) except: raise finally: client.stop() received = [] for r in receivers: received.extend(r.receive(timeout=1)) assert len(received) == 1 assert list(received[0].body) == [b"A", b"B", b"C"]
def test_send_with_long_interval_sync(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubClient.from_connection_string(connection_str, debug=True) sender = client.add_sender() try: client.run() sender.send(EventData(b"A single event")) for _ in range(2): time.sleep(300) sender.send(EventData(b"A single event")) finally: client.stop() received = [] for r in receivers: received.extend(r.receive(timeout=1)) assert len(received) == 3 assert list(received[0].body)[0] == b"A single event"
def test_long_running_receive(connection_str): parser = argparse.ArgumentParser() parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30) parser.add_argument("--consumer", help="Consumer group name", default="$default") parser.add_argument("--partitions", help="Comma seperated partition IDs") parser.add_argument("--offset", help="Starting offset", default="-1") parser.add_argument("--conn-str", help="EventHub connection string", default=connection_str) parser.add_argument("--eventhub", help="Name of EventHub") parser.add_argument("--address", help="Address URI to the EventHub entity") parser.add_argument("--sas-policy", help="Name of the shared access policy to authenticate with") parser.add_argument("--sas-key", help="Shared access key") args, _ = parser.parse_known_args() if args.conn_str: client = EventHubClient.from_connection_string( args.conn_str, eventhub=args.eventhub, debug=False) elif args.address: client = EventHubClient( args.address, username=args.sas_policy, password=args.sas_key) else: try: import pytest pytest.skip("Must specify either '--conn-str' or '--address'") except ImportError: raise ValueError("Must specify either '--conn-str' or '--address'") try: if not args.partitions: partitions = get_partitions(client) else: partitions = args.partitions.split(",") pumps = {} for pid in partitions: pumps[pid] = client.add_receiver( consumer_group=args.consumer, partition=pid, offset=Offset(args.offset), prefetch=50) client.run() pump(pumps, args.duration) finally: client.stop()
def test_send_and_receive_large_body_size(connstr_receivers): if sys.platform.startswith('darwin'): pytest.skip("Skipping on OSX - open issue regarding message size") connection_str, receivers = connstr_receivers client = EventHubClient.from_connection_string(connection_str, debug=False) sender = client.add_sender() try: client.run() payload = 250 * 1024 sender.send(EventData("A" * payload)) except: raise finally: client.stop() received = [] for r in receivers: received.extend(r.receive(timeout=4)) assert len(received) == 1 assert len(list(received[0].body)[0]) == payload
def test_send_partition_batch(connstr_receivers): connection_str, receivers = connstr_receivers def batched(): for i in range(10): yield "Event number {}".format(i) client = EventHubClient.from_connection_string(connection_str, debug=False) sender = client.add_sender(partition="1") try: client.run() sender.send(EventData(batch=batched())) time.sleep(1) except: raise finally: client.stop() partition_0 = receivers[0].receive(timeout=2) assert len(partition_0) == 0 partition_1 = receivers[1].receive(timeout=2) assert len(partition_1) == 10
def test_example_eventhub_transfer(connection_str): import os from azure.eventhub import EventHubClient, EventData client = EventHubClient.from_connection_string(connection_str) sender = client.add_sender() try: client.run() # [START eventhub_client_transfer] logger = logging.getLogger("azure.eventhub") def callback(outcome, condition): logger.info("Message sent. Outcome: {}, Condition: {}".format( outcome, condition)) event_data = EventData(b"A single event") sender.transfer(event_data, callback=callback) sender.wait() # [END eventhub_client_transfer] except: raise finally: client.stop()
def test_send_with_forced_conn_close_sync(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubClient.from_connection_string(connection_str, debug=True) sender = client.add_sender() try: client.run() sender.send(EventData(b"A single event")) sender._handler._message_sender.destroy() time.sleep(300) sender.send(EventData(b"A single event")) sender.send(EventData(b"A single event")) sender._handler._message_sender.destroy() time.sleep(300) sender.send(EventData(b"A single event")) sender.send(EventData(b"A single event")) finally: client.stop() received = [] for r in receivers: received.extend(r.receive(timeout=1)) assert len(received) == 5 assert list(received[0].body)[0] == b"A single event"
def test_send_batch_with_app_prop_sync(connstr_receivers): pytest.skip("Waiting on uAMQP release") connection_str, receivers = connstr_receivers def batched(): for i in range(10): yield "Event number {}".format(i) for i in range(10, 20): yield EventData("Event number {}".format(i)) client = EventHubClient.from_connection_string(connection_str, debug=False) sender = client.add_sender() try: client.run() app_prop_key = "raw_prop" app_prop_value = "raw_value" batch_app_prop = {app_prop_key:app_prop_value} batch_event = EventData(batch=batched()) batch_event.application_properties = batch_app_prop sender.send(batch_event) except: raise finally: client.stop() time.sleep(1) received = [] for r in receivers: received.extend(r.receive(timeout=3)) assert len(received) == 20 for index, message in enumerate(received): assert list(message.body)[0] == "Event number {}".format(index).encode('utf-8') assert (app_prop_key.encode('utf-8') in message.application_properties) \ and (dict(message.application_properties)[app_prop_key.encode('utf-8')] == app_prop_value.encode('utf-8'))
def test_send_batch_sync(connstr_receivers): connection_str, receivers = connstr_receivers def batched(): for i in range(10): yield "Event number {}".format(i) client = EventHubClient.from_connection_string(connection_str, debug=False) sender = client.add_sender() try: client.run() sender.send(EventData(batch=batched())) except: raise finally: client.stop() time.sleep(1) received = [] for r in receivers: received.extend(r.receive(timeout=3)) assert len(received) == 10 for index, message in enumerate(received): assert list(message.body)[0] == "Event number {}".format(index).encode('utf-8')
# SAS policy and key are not required if they are encoded in the URL USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') def callback(outcome, condition): logger.info("Message sent. Outcome: {}, Condition: {}".format( outcome, condition)) try: if not ADDRESS: raise ValueError("No EventHubs URL supplied.") client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) sender = client.add_sender(partition="1") client.run() try: start_time = time.time() for i in range(100): sender.transfer(EventData(str(i)), callback=callback) logger.info("Queued 100 messages.") sender.wait() logger.info("Finished processing queue.") except: raise finally: end_time = time.time() client.stop() run_time = end_time - start_time