async def open_clients_async(self):
"""
Responsible for establishing connection to event hub client
throws EventHubsException, IOException, InterruptedException, ExecutionException
"""
await self.partition_context.get_initial_offset_async()
# Create event hub client and receive handler and set options
self.partition_receive_handler = AsyncReceiver(
loop=self.loop, prefetch=self.host.eph_options.prefetch_count)
self.eh_client = EventHubClient(self.host.eh_config.client_address) \
.subscribe(self.partition_receive_handler,
self.partition_context.consumer_group_name,
self.partition_context.partition_id,
Offset(self.partition_context.offset))
self.partition_receiver = PartitionReceiver(self)
def receiveEvents():
try:
ADDRESS = "amqps://iothubowner%40sas.root.pi-broker:SharedAccessSignature%20sr%3Dpi-broker.azure-devices.net%26sig%3DMSg9Pba4uObLCa%252B4CcR3tuN2KuTOtpz1KBiN3Vsk6jI%253D%26skn%3Diothubowner%26se%[email protected]/messages/events/"
CONSUMER_GROUP = "python-webapp"
OFFSET = Offset("-1")
EventHubClient(ADDRESS if len(sys.argv) == 1 else sys.argv[1]) \
.subscribe(MyReceiver("0"), CONSUMER_GROUP, "0", OFFSET) \
.run()
except KeyboardInterrupt:
pass
def on_subscription_data(self, data):
ADDRESS = ("amqps://"
"send"
":"
"RJQLhw9Uu8qHL0KZQkb0dd+4didTvGqIzJZljtP5m/s="
"@"
"structstreaming.servicebus.windows.net"
"/"
"twitter")
sender = Sender()
azClient = EventHubClient(ADDRESS if len(sys.argv) == 1 else sys.argv[1]) \
.publish(sender) \
.run_daemon()
for message in data['messages']:
sender.send(EventData(str(message)))
print("Got message:", message)
async def send(snd, count):
for i in range(count):
await snd.send(EventData(str(i)))
logger.info("Send message %d", i)
try:
ADDRESS = ("amqps://"
"<SAS-policy>"
":"
"<SAS-key>"
"@"
"<mynamespace>.servicebus.windows.net"
"/"
"myeventhub")
sender = AsyncSender()
client = EventHubClient(ADDRESS if len(sys.argv) == 1 else sys.argv[1]) \
.publish(sender) \
.run_daemon()
loop = asyncio.get_event_loop()
loop.run_until_complete(send(sender, 100))
client.stop()
loop.close()
except KeyboardInterrupt:
pass
class EventHubPartitionPump(PartitionPump):
"""
Pulls and messages from lease partition from eventhub and sends them to processor
"""
def __init__(self, host, lease):
PartitionPump.__init__(self, host, lease)
self.eh_client = None
self.partition_receiver = None
self.partition_receive_handler = None
async def on_open_async(self):
"""
Eventhub Override for on_open_async
"""
_opened_ok = False
_retry_count = 0
while (not _opened_ok) and (_retry_count < 5):
try:
await self.open_clients_async()
_opened_ok = True
except Exception as err:
logging.warning(
"%s,%s PartitionPumpWarning: Failure creating client or receiver,\
retrying: %s", self.host.guid,
self.partition_context.partition_id, repr(err))
last_exception = err
_retry_count += 1
if not _opened_ok:
await self.processor.process_error_async(self.partition_context,
last_exception)
self.set_pump_status("OpenFailed")
if self.pump_status == "Opening":
self.set_pump_status("Running")
self.eh_client.run_daemon()
await self.partition_receiver.run()
if self.pump_status == "OpenFailed":
self.set_pump_status("Closing")
await self.clean_up_clients_async()
self.set_pump_status("Closed")
async def open_clients_async(self):
"""
Responsible for establishing connection to event hub client
throws EventHubsException, IOException, InterruptedException, ExecutionException
"""
await self.partition_context.get_initial_offset_async()
# Create event hub client and receive handler and set options
self.partition_receive_handler = AsyncReceiver(
loop=self.loop, prefetch=self.host.eph_options.prefetch_count)
self.eh_client = EventHubClient(self.host.eh_config.client_address) \
.subscribe(self.partition_receive_handler,
self.partition_context.consumer_group_name,
self.partition_context.partition_id,
Offset(self.partition_context.offset))
self.partition_receiver = PartitionReceiver(self)
async def clean_up_clients_async(self):
"""
Resets the pump swallows all exceptions
"""
if self.partition_receiver:
if self.eh_client:
self.eh_client.stop()
self.partition_receiver = None
self.partition_receive_handler = None
self.eh_client = None
async def on_closing_async(self, reason):
"""
Overides partition pump on cleasing
"""
await self.clean_up_clients_async()
self.last_offset = EventData.offset(message)
self.last_sn = EventData.sequence_number(message)
# message.properties (dict) - application defined properties
# message.body (object) - application set object
self.total += 1
if self.total % 50 == 0:
# do checkpoint for every 50 events received
logging.info("Received %s, sn=%d offset=%s",
self.total,
self.last_sn,
self.last_offset)
try:
logging.basicConfig(filename="test.log", level=logging.INFO)
logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))
ADDRESS = ("amqps://"
"<URL-encoded-SAS-policy>"
":"
"<URL-encoded-SAS-key>"
"@"
"<mynamespace>.servicebus.windows.net"
"/"
"myeventhub")
CONSUMER_GROUP = "$default"
PARTITION = "1"
OFFSET = "-1"
EventHubClient(ADDRESS).subscribe(MyReceiver(), CONSUMER_GROUP, PARTITION, OFFSET).run()
except KeyboardInterrupt:
pass
self.total = 0
self.last_sn = -1
self.last_offset = "-1"
def on_event_data(self, event_data):
self.last_offset = event_data.offset
self.last_sn = event_data.sequence_number
self.total += 1
logger.info("Partition %s, Received %s, sn=%d offset=%s",
self.partition, self.total, self.last_sn, self.last_offset)
try:
ADDRESS = ("amqps://"
"<SAS-policy>"
":"
"<SAS-key>"
"@"
"<mynamespace>.servicebus.windows.net"
"/"
"myeventhub")
CONSUMER_GROUP = "$default"
OFFSET = Offset("-1")
EventHubClient(ADDRESS if len(sys.argv) == 1 else sys.argv[1]) \
.subscribe(MyReceiver("0"), CONSUMER_GROUP, "0", OFFSET) \
.run()
except KeyboardInterrupt:
pass
self._failure += 1
else:
self._success += 1
if self._total % 500 == 0:
logger.info("Send total %d, success %d, failure %d",
self._total,
self._success,
self._failure)
if time.time() < self._deadline:
self.start_send()
else:
self._event.set()
try:
sender = Sender()
client = EventHubClient(args.address).publish(sender).run_daemon()
if args.batch > 1:
TransferClient(sender).run()
else:
total = 0
deadline = time.time() + args.duration
while time.time() < deadline:
try:
sender.send(EventData("D" * args.payload))
total += 1
if total % 500 == 0:
logger.info("Send total %d", total)
except Exception as err:
logger.error("Send failed %s", err)
client.stop()
except KeyboardInterrupt:
type=int,
default=3600)
parser.add_argument("--consumer",
help="Consumer group name",
default="$default")
parser.add_argument("--partitions",
help="Comma seperated partition IDs",
default="0")
parser.add_argument("--offset", help="Starting offset", default="-1")
parser.add_argument("address",
help="Address Uri to the event hub entity",
nargs="?")
try:
args = parser.parse_args()
loop = asyncio.get_event_loop()
client = EventHubClient(args.address)
deadline = time.time() + args.duration
asyncio.gather()
pumps = []
for pid in args.partitions.split(","):
receiver = AsyncReceiver()
client.subscribe(receiver, args.consumer, pid, Offset(args.offset))
pumps.append(pump(pid, receiver, deadline))
client.run_daemon()
loop.run_until_complete(asyncio.gather(*pumps))
client.stop()
loop.close()
except KeyboardInterrupt:
pass
# simulate an async event processing
await asyncio.sleep(0.05)
except asyncio.TimeoutError:
logger.info("No events received, queue size %d, delivered %d", recv.messages.qsize(), recv.delivered)
try:
ADDRESS = ("amqps://"
"<URL-encoded-SAS-policy>"
":"
"<URL-encoded-SAS-key>"
"@"
"<mynamespace>.servicebus.windows.net"
"/"
"myeventhub")
CONSUMER_GROUP = "$default"
OFFSET = Offset("-1")
logger.info("starting loop")
loop = asyncio.get_event_loop()
receiver = AsyncReceiver()
client = EventHubClient(ADDRESS if len(sys.argv) == 1 else sys.argv[1]) \
.subscribe(receiver, CONSUMER_GROUP, "0", OFFSET) \
.run_daemon()
loop.run_until_complete(pump(receiver, 1000))
client.stop()
loop.close()
except KeyboardInterrupt:
pass