def test_send_with_partition_key(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubProducerClient.from_connection_string(connection_str) with client: data_val = 0 for partition in [b"a", b"b", b"c", b"d", b"e", b"f"]: partition_key = b"test_partition_" + partition for i in range(50): batch = client.create_batch(partition_key=partition_key) batch.add(EventData(str(data_val))) data_val += 1 client.send_batch(batch) client.send_batch(client.create_batch()) found_partition_keys = {} for index, partition in enumerate(receivers): received = partition.receive_message_batch(timeout=5000) for message in received: try: event_data = EventData._from_message(message) existing = found_partition_keys[event_data.partition_key] assert existing == index except KeyError: found_partition_keys[event_data.partition_key] = index
def test_send_batch_pid_pk(invalid_hostname, partition_id, partition_key): # Use invalid_hostname because this is not a live test. client = EventHubProducerClient.from_connection_string(invalid_hostname) batch = EventDataBatch(partition_id=partition_id, partition_key=partition_key) with client: with pytest.raises(TypeError): client.send_batch(batch, partition_id=partition_id, partition_key=partition_key)
def test_send_partition(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubProducerClient.from_connection_string(connection_str) with client: batch = client.create_batch() batch.add(EventData(b"Data")) client.send_batch(batch) with client: batch = client.create_batch(partition_id="1") batch.add(EventData(b"Data")) client.send_batch(batch) partition_0 = receivers[0].receive_message_batch(timeout=5000) partition_1 = receivers[1].receive_message_batch(timeout=5000) assert len(partition_0) + len(partition_1) == 2 with client: batch = client.create_batch() batch.add(EventData(b"Data")) client.send_batch(batch) with client: batch = client.create_batch(partition_id="1") batch.add(EventData(b"Data")) client.send_batch(batch) partition_0 = receivers[0].receive_message_batch(timeout=5000) partition_1 = receivers[1].receive_message_batch(timeout=5000) assert len(partition_0) + len(partition_1) == 2
def __init__(self, opts): self._opts = opts self._client = None self._mode = None self._state = None self._verbose = False conn_str = opts['conn_str'] hub_name = opts['hub_name'] if 'verbose' in opts: if opts['verbose'] == True: self._verbose = True if self.is_consumer(): consumer_group = opts['consumer_group'] self._client = EventHubConsumerClient.from_connection_string( conn_str, consumer_group, eventhub_name=hub_name) self._mode = 'consumer' self._state = 'open' else: self._client = EventHubProducerClient.from_connection_string( conn_str, eventhub_name=hub_name) self._mode = 'producer' self._state = 'open' if self._verbose: print("EventHub.__init__ client: {}".format(str(type( self._client))))
def stress_send_sync(producer: EventHubProducerClient, args, logger): batch = producer.create_batch(partition_id=args.send_partition_id, partition_key=args.send_partition_key) try: while True: event_data = EventData(body=b"D" * args.payload) batch.add(event_data) except ValueError: try: producer.send_batch(batch) except EventHubError as e: if args.ignore_send_failure: logger.warning("Sync send failed due to error: %r.", e) return 0 raise return len(batch)
def home(): response = requests.get(settings.API_URL + '/getAdvertisements') response2 = requests.get(settings.API_URL + '/getPosts') ads = response.json() posts = response2.json() # code to use LogicApp 'logicapphttptrigger'. Send and email. payload = { "time": datetime.now().strftime("%d/%m/%Y %H:%M:%S"), "event": "Main Page Request" } response3 = requests.post(os.environ['LOGIC_APP_URL'], json=payload) # code to generate an Event and to send it to the Event Hub. producer = EventHubProducerClient.from_connection_string( conn_str= "Endpoint=sb://namespaceneighborlyapp.servicebus.windows.net/;SharedAccessKeyName=neighborlyhubsend;SharedAccessKey=RPBuDZ9OUi+xTxdMy1AixwpMfvAt64h1FlkzBJ+rq+Y=;EntityPath=eventhubneighborlyapp", eventhub_name="eventhubneighborlyapp") reading = { 'id': 'HOME PAGE', 'topic': 'ACTIVITY', 'subject': 'HOME PAGE ACTIVITY', 'event_type': 'EVENT HUB TRIGGER' } s = json.dumps(reading) # Convert the reading into a JSON string. event_data_batch = producer.create_batch() event_data_batch.add(EventData(s)) # Add event data to the batch. producer.send_batch( event_data_batch) # Send the batch of events to the event hub. producer.close() return render_template("index.html", ads=ads, posts=posts)
def test_send_connection_idle_timeout_and_reconnect_sync(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubProducerClient.from_connection_string( conn_str=connection_str, idle_timeout=10) with client: ed = EventData('data') sender = client._create_producer(partition_id='0') with sender: sender._open_with_retry() time.sleep(11) sender._unsent_events = [ed.message] ed.message.on_send_complete = sender._on_outcome with pytest.raises( (uamqp.errors.ConnectionClose, uamqp.errors.MessageHandlerError, OperationTimeoutError)): # Mac may raise OperationTimeoutError or MessageHandlerError sender._send_event_data() sender._send_event_data_with_retry() retry = 0 while retry < 3: try: messages = receivers[0].receive_message_batch(max_batch_size=10, timeout=10000) if messages: received_ed1 = EventData._from_message(messages[0]) assert received_ed1.body_as_str() == 'data' break except compat.TimeoutException: retry += 1
def test_send_multiple_partitions_with_app_prop(connstr_receivers): connection_str, receivers = connstr_receivers app_prop_key = "raw_prop" app_prop_value = "raw_value" app_prop = {app_prop_key: app_prop_value} client = EventHubProducerClient.from_connection_string(connection_str) with client: ed0 = EventData(b"Message 0") ed0.properties = app_prop batch = client.create_batch(partition_id="0") batch.add(ed0) client.send_batch(batch) ed1 = EventData(b"Message 1") ed1.properties = app_prop batch = client.create_batch(partition_id="1") batch.add(ed1) client.send_batch(batch) partition_0 = [ EventData._from_message(x) for x in receivers[0].receive_message_batch(timeout=5000) ] assert len(partition_0) == 1 assert partition_0[0].properties[b"raw_prop"] == b"raw_value" partition_1 = [ EventData._from_message(x) for x in receivers[1].receive_message_batch(timeout=5000) ] assert len(partition_1) == 1 assert partition_1[0].properties[b"raw_prop"] == b"raw_value"
def example_eventhub_producer_ops(): # [START eventhub_producer_client_close_sync] import os from azure.eventhub import EventHubProducerClient, EventData event_hub_connection_str = os.environ['EVENT_HUB_CONN_STR'] eventhub_name = os.environ['EVENT_HUB_NAME'] producer = EventHubProducerClient.from_connection_string( conn_str=event_hub_connection_str, eventhub_name=eventhub_name) try: event_data_batch = producer.create_batch(max_size_in_bytes=10000) while True: try: event_data_batch.add( EventData('Message inside EventBatchData')) except ValueError: # EventDataBatch object reaches max_size. # New EventDataBatch object can be created here to send more data break producer.send_batch(event_data_batch) finally: # Close down the producer handler. producer.close()
def test_send_null_body(connection_str): client = EventHubProducerClient.from_connection_string(connection_str) try: with pytest.raises(ValueError): data = EventData(None) client.send(data) finally: client.close()
def test_send_with_invalid_key(live_eventhub): conn_str = live_eventhub["connection_str"].format( live_eventhub['hostname'], live_eventhub['key_name'], 'invalid', live_eventhub['event_hub']) client = EventHubProducerClient.from_connection_string(conn_str) with pytest.raises(ConnectError): client.send(EventData("test data")) client.close()
def test_send_with_invalid_hostname(invalid_hostname): if sys.platform.startswith('darwin'): pytest.skip("Skipping on OSX - it keeps reporting 'Unable to set external certificates' " "and blocking other tests") client = EventHubProducerClient.from_connection_string(invalid_hostname) with client: with pytest.raises(ConnectError): client.send(EventData("test data"))
def test_create_batch_with_invalid_hostname_sync(invalid_hostname): if sys.platform.startswith('darwin'): pytest.skip("Skipping on OSX - it keeps reporting 'Unable to set external certificates' " "and blocking other tests") client = EventHubProducerClient.from_connection_string(invalid_hostname) with client: with pytest.raises(ConnectError): client.create_batch(max_size=300)
def test_send_batch_with_invalid_key(invalid_key): client = EventHubProducerClient.from_connection_string(invalid_key) try: with pytest.raises(ConnectError): batch = EventDataBatch() batch.add(EventData("test data")) client.send_batch(batch) finally: client.close()
def test_send_list_partition(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubProducerClient.from_connection_string(connection_str) payload = "A1" with client: client.send_batch([EventData(payload)], partition_id="0") message = receivers[0].receive_message_batch(timeout=10000)[0] received = EventData._from_message(message) assert received.body_as_str() == payload
def send_event_data_batch(producer: EventHubProducerClient, data: str): """Send/publish data to events hub syncrounously Without specifying partition_id or partition_key the events will be distributed to available partitions via round-robin. Parameters ---------- producer: the eventhub client data: data to be published in string format Returns ------- none """ event_data_batch = producer.create_batch(partition_id="0") event_data_batch.add(EventData(data)) producer.send_batch(event_data_batch)
def __init__(self, arguments): super().__init__(arguments) connection_string = self.get_from_env( "AZURE_EVENTHUB_CONNECTION_STRING") eventhub_name = self.get_from_env("AZURE_EVENTHUB_NAME") self.producer = EventHubProducerClient.from_connection_string( connection_string, eventhub_name=eventhub_name) self.async_producer = AsyncEventHubProducerClient.from_connection_string( connection_string, eventhub_name=eventhub_name)
def send_event_data_batch_with_partition_id(producer: EventHubProducerClient, data: str, pid: int): """Send/publish data to events hub syncrounously Specifying partition_key Parameters ---------- producer: the eventhub client data: data to be published in string format pid: partion key string Returns ------- none """ event_data_batch_with_partition_id = producer.create_batch( partition_id=str(pid)) event_data_batch_with_partition_id.add(EventData(data)) producer.send_batch(event_data_batch_with_partition_id)
def ii_init(self, record_info_in: Sdk.RecordInfo) -> bool: self.Info = record_info_in self.Producer = EventHubProducerClient.from_connection_string( conn_str=self.parent.EventHubsConnStr, eventhub_name=self.parent.EventHubName) for field in self.Info: self.DataMap[field.name] = None self.parent.display_info_msg("Event Hubs producer client created") return True
def test_send_to_invalid_partitions(connection_str): partitions = ["XYZ", "-1", "1000", "-"] for p in partitions: client = EventHubProducerClient.from_connection_string(connection_str) try: with pytest.raises(ConnectError): client.send(EventData("test data"), partition_id=p) finally: client.close()
def test_send_with_long_interval_sync(live_eventhub, sleep): test_partition = "0" sender = EventHubProducerClient( live_eventhub['hostname'], live_eventhub['event_hub'], EventHubSharedKeyCredential(live_eventhub['key_name'], live_eventhub['access_key'])) with sender: batch = sender.create_batch(partition_id=test_partition) batch.add(EventData(b"A single event")) sender.send_batch(batch) if sleep: time.sleep(250) else: sender._producers[ test_partition]._handler._connection._conn.destroy() batch = sender.create_batch(partition_id=test_partition) batch.add(EventData(b"A single event")) sender.send_batch(batch) received = [] uri = "sb://{}/{}".format(live_eventhub['hostname'], live_eventhub['event_hub']) sas_auth = authentication.SASTokenAuth.from_shared_access_key( uri, live_eventhub['key_name'], live_eventhub['access_key']) source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format( live_eventhub['hostname'], live_eventhub['event_hub'], live_eventhub['consumer_group'], test_partition) receiver = uamqp.ReceiveClient(source, auth=sas_auth, debug=False, timeout=5000, prefetch=500) try: receiver.open() # receive_message_batch() returns immediately once it receives any messages before the max_batch_size # and timeout reach. Could be 1, 2, or any number between 1 and max_batch_size. # So call it twice to ensure the two events are received. received.extend([ EventData._from_message(x) for x in receiver.receive_message_batch(max_batch_size=1, timeout=5000) ]) received.extend([ EventData._from_message(x) for x in receiver.receive_message_batch(max_batch_size=1, timeout=5000) ]) finally: receiver.close() assert len(received) == 2 assert list(received[0].body)[0] == b"A single event"
def EventhubSender(eventhubAppCredential): print("Initiating sender client..") #Creating a Eventhub producer client using ServicePrincipal credentials producer = EventHubProducerClient(fully_qualified_namespace=fully_qualified_namespace, eventhub_name=eventhub_name, credential=eventhubAppCredential) with producer: event_data_batch = producer.create_batch() try: event_data_batch.add(EventData('Message inside EventBatchData')) except ValueError: print("error") producer.send_batch(event_data_batch) print('One Batch sent successfully.')
def send_once(): client = EventHubProducerClient.from_connection_string( connection_str, eventhub_name=eventhub_name ) with client: event_data = client.create_batch() event_data.add(EventData('{"film":"Avengers", "rank": "3"}')) client.send_batch(event_data)
def test_send_partition(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubProducerClient.from_connection_string(connection_str) with client: client.send(EventData(b"Data"), partition_id="1") partition_0 = receivers[0].receive_message_batch(timeout=5000) assert len(partition_0) == 0 partition_1 = receivers[1].receive_message_batch(timeout=5000) assert len(partition_1) == 1
def test_send_batch_null_body(connection_str): client = EventHubProducerClient.from_connection_string(connection_str) try: with pytest.raises(ValueError): data = EventData(None) batch = client.create_batch() batch.add(data) client.send_batch(batch) finally: client.close()
def send_event_data_batch_with_properties(producer: EventHubProducerClient, data: str, properties: Dict[str, str]): """Send/publish data to events hub syncrounously Parameters ---------- producer: the eventhub client data: data to be published in string format properties: a dictionary of key=value of data to be added to the message Returns ------- none """ event_data_batch = producer.create_batch() event_data = EventData(data) event_data.properties = properties event_data_batch.add(event_data) producer.send_batch(event_data_batch)
def test_send_too_large_message(connection_str): if sys.platform.startswith('darwin'): pytest.skip("Skipping on OSX - open issue regarding message size") client = EventHubProducerClient.from_connection_string(connection_str) try: data = EventData(b"A" * 1100000) with pytest.raises(EventDataSendError): client.send(data) finally: client.close()
def test_client_secret_credential(aad_credential, live_eventhub): try: from azure.identity import EnvironmentCredential except: pytest.skip("No azure identity library") credential = EnvironmentCredential() producer_client = EventHubProducerClient( fully_qualified_namespace=live_eventhub['hostname'], eventhub_name=live_eventhub['event_hub'], credential=credential, user_agent='customized information') consumer_client = EventHubConsumerClient( fully_qualified_namespace=live_eventhub['hostname'], eventhub_name=live_eventhub['event_hub'], consumer_group='$default', credential=credential, user_agent='customized information') with producer_client: batch = producer_client.create_batch(partition_id='0') batch.add(EventData(body='A single message')) producer_client.send_batch(batch) def on_event(partition_context, event): on_event.called = True on_event.partition_id = partition_context.partition_id on_event.event = event on_event.called = False with consumer_client: worker = threading.Thread(target=consumer_client.receive, args=(on_event, ), kwargs={ "partition_id": '0', "starting_position": '-1' }) worker.start() time.sleep(13) worker.join() assert on_event.called is True assert on_event.partition_id == "0" assert list(on_event.event.body)[0] == 'A single message'.encode('utf-8')
def test_send_and_receive_small_body(connstr_receivers, payload): connection_str, receivers = connstr_receivers client = EventHubProducerClient.from_connection_string(connection_str) with client: client.send(EventData(payload)) received = [] for r in receivers: received.extend([EventData._from_message(x) for x in r.receive_message_batch(timeout=5000)]) assert len(received) == 1 assert list(received[0].body)[0] == payload
def test_send_non_ascii(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubProducerClient.from_connection_string(connection_str) with client: client.send(EventData(u"é,è,à,ù,â,ê,î,ô,û"), partition_id="0") client.send(EventData(json.dumps({"foo": u"漢字"})), partition_id="0") time.sleep(1) partition_0 = [EventData._from_message(x) for x in receivers[0].receive_message_batch(timeout=5000)] assert len(partition_0) == 2 assert partition_0[0].body_as_str() == u"é,è,à,ù,â,ê,î,ô,û" assert partition_0[1].body_as_json() == {"foo": u"漢字"}