Example #1
0
 async def open_clients_async(self):
     """
     Responsible for establishing connection to event hub client
     throws EventHubsException, IOException, InterruptedException, ExecutionException
     """
     await self.partition_context.get_initial_offset_async()
     # Create event hub client and receive handler and set options
     self.partition_receive_handler = AsyncReceiver(
         loop=self.loop, prefetch=self.host.eph_options.prefetch_count)
     self.eh_client = EventHubClient(self.host.eh_config.client_address) \
                     .subscribe(self.partition_receive_handler,
                                self.partition_context.consumer_group_name,
                                self.partition_context.partition_id,
                                Offset(self.partition_context.offset))
     self.partition_receiver = PartitionReceiver(self)
Example #2
0
def receiveEvents():
    try:
        ADDRESS = "amqps://iothubowner%40sas.root.pi-broker:SharedAccessSignature%20sr%3Dpi-broker.azure-devices.net%26sig%3DMSg9Pba4uObLCa%252B4CcR3tuN2KuTOtpz1KBiN3Vsk6jI%253D%26skn%3Diothubowner%26se%[email protected]/messages/events/"
        CONSUMER_GROUP = "python-webapp"
        OFFSET = Offset("-1")

        EventHubClient(ADDRESS if len(sys.argv) == 1 else sys.argv[1]) \
            .subscribe(MyReceiver("0"), CONSUMER_GROUP, "0", OFFSET) \
            .run()

    except KeyboardInterrupt:
        pass
Example #3
0
            def on_subscription_data(self, data):
                ADDRESS = ("amqps://"
                           "send"
                           ":"
                           "RJQLhw9Uu8qHL0KZQkb0dd+4didTvGqIzJZljtP5m/s="
                           "@"
                           "structstreaming.servicebus.windows.net"
                           "/"
                           "twitter")
                sender = Sender()
                azClient = EventHubClient(ADDRESS if len(sys.argv) == 1 else sys.argv[1]) \
                             .publish(sender) \
                             .run_daemon()

                for message in data['messages']:
                    sender.send(EventData(str(message)))
                    print("Got message:", message)

async def send(snd, count):
    for i in range(count):
        await snd.send(EventData(str(i)))
        logger.info("Send message %d", i)


try:
    ADDRESS = ("amqps://"
               "<SAS-policy>"
               ":"
               "<SAS-key>"
               "@"
               "<mynamespace>.servicebus.windows.net"
               "/"
               "myeventhub")

    sender = AsyncSender()
    client = EventHubClient(ADDRESS if len(sys.argv) == 1 else sys.argv[1]) \
                 .publish(sender) \
                 .run_daemon()

    loop = asyncio.get_event_loop()
    loop.run_until_complete(send(sender, 100))
    client.stop()
    loop.close()

except KeyboardInterrupt:
    pass
Example #5
0
class EventHubPartitionPump(PartitionPump):
    """
    Pulls and messages from lease partition from eventhub and sends them to processor
    """
    def __init__(self, host, lease):
        PartitionPump.__init__(self, host, lease)
        self.eh_client = None
        self.partition_receiver = None
        self.partition_receive_handler = None

    async def on_open_async(self):
        """
        Eventhub Override for on_open_async
        """
        _opened_ok = False
        _retry_count = 0
        while (not _opened_ok) and (_retry_count < 5):
            try:
                await self.open_clients_async()
                _opened_ok = True
            except Exception as err:
                logging.warning(
                    "%s,%s PartitionPumpWarning: Failure creating client or receiver,\
                                retrying: %s", self.host.guid,
                    self.partition_context.partition_id, repr(err))
                last_exception = err
                _retry_count += 1

        if not _opened_ok:
            await self.processor.process_error_async(self.partition_context,
                                                     last_exception)
            self.set_pump_status("OpenFailed")

        if self.pump_status == "Opening":
            self.set_pump_status("Running")
            self.eh_client.run_daemon()
            await self.partition_receiver.run()

        if self.pump_status == "OpenFailed":
            self.set_pump_status("Closing")
            await self.clean_up_clients_async()
            self.set_pump_status("Closed")

    async def open_clients_async(self):
        """
        Responsible for establishing connection to event hub client
        throws EventHubsException, IOException, InterruptedException, ExecutionException
        """
        await self.partition_context.get_initial_offset_async()
        # Create event hub client and receive handler and set options
        self.partition_receive_handler = AsyncReceiver(
            loop=self.loop, prefetch=self.host.eph_options.prefetch_count)
        self.eh_client = EventHubClient(self.host.eh_config.client_address) \
                        .subscribe(self.partition_receive_handler,
                                   self.partition_context.consumer_group_name,
                                   self.partition_context.partition_id,
                                   Offset(self.partition_context.offset))
        self.partition_receiver = PartitionReceiver(self)

    async def clean_up_clients_async(self):
        """
        Resets the pump swallows all exceptions
        """
        if self.partition_receiver:
            if self.eh_client:
                self.eh_client.stop()
                self.partition_receiver = None
                self.partition_receive_handler = None
                self.eh_client = None

    async def on_closing_async(self, reason):
        """
        Overides partition pump on cleasing
        """
        await self.clean_up_clients_async()
        self.last_offset = EventData.offset(message)
        self.last_sn = EventData.sequence_number(message)
        # message.properties (dict) - application defined properties
        # message.body (object) - application set object
        self.total += 1
        if self.total % 50 == 0:
            # do checkpoint for every 50 events received
            logging.info("Received %s, sn=%d offset=%s",
                         self.total,
                         self.last_sn,
                         self.last_offset)

try:
    logging.basicConfig(filename="test.log", level=logging.INFO)
    logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))

    ADDRESS = ("amqps://"
               "<URL-encoded-SAS-policy>"
               ":"
               "<URL-encoded-SAS-key>"
               "@"
               "<mynamespace>.servicebus.windows.net"
               "/"
               "myeventhub")
    CONSUMER_GROUP = "$default"
    PARTITION = "1"
    OFFSET = "-1"
    EventHubClient(ADDRESS).subscribe(MyReceiver(), CONSUMER_GROUP, PARTITION, OFFSET).run()
except KeyboardInterrupt:
    pass
Example #7
0
        self.total = 0
        self.last_sn = -1
        self.last_offset = "-1"

    def on_event_data(self, event_data):
        self.last_offset = event_data.offset
        self.last_sn = event_data.sequence_number
        self.total += 1
        logger.info("Partition %s, Received %s, sn=%d offset=%s",
                    self.partition, self.total, self.last_sn, self.last_offset)


try:
    ADDRESS = ("amqps://"
               "<SAS-policy>"
               ":"
               "<SAS-key>"
               "@"
               "<mynamespace>.servicebus.windows.net"
               "/"
               "myeventhub")
    CONSUMER_GROUP = "$default"
    OFFSET = Offset("-1")

    EventHubClient(ADDRESS if len(sys.argv) == 1 else sys.argv[1]) \
        .subscribe(MyReceiver("0"), CONSUMER_GROUP, "0", OFFSET) \
        .run()

except KeyboardInterrupt:
    pass
Example #8
0
            self._failure += 1
        else:
            self._success += 1
        if self._total % 500 == 0:
            logger.info("Send total %d, success %d, failure %d",
                        self._total,
                        self._success,
                        self._failure)
        if time.time() < self._deadline:
            self.start_send()
        else:
            self._event.set()

try:
    sender = Sender()
    client = EventHubClient(args.address).publish(sender).run_daemon()
    if args.batch > 1:
        TransferClient(sender).run()
    else:
        total = 0
        deadline = time.time() + args.duration
        while time.time() < deadline:
            try:
                sender.send(EventData("D" * args.payload))
                total += 1
                if total % 500 == 0:
                    logger.info("Send total %d", total)
            except Exception as err:
                logger.error("Send failed %s", err)
    client.stop()
except KeyboardInterrupt:
Example #9
0
                    type=int,
                    default=3600)
parser.add_argument("--consumer",
                    help="Consumer group name",
                    default="$default")
parser.add_argument("--partitions",
                    help="Comma seperated partition IDs",
                    default="0")
parser.add_argument("--offset", help="Starting offset", default="-1")
parser.add_argument("address",
                    help="Address Uri to the event hub entity",
                    nargs="?")

try:
    args = parser.parse_args()
    loop = asyncio.get_event_loop()
    client = EventHubClient(args.address)
    deadline = time.time() + args.duration
    asyncio.gather()
    pumps = []
    for pid in args.partitions.split(","):
        receiver = AsyncReceiver()
        client.subscribe(receiver, args.consumer, pid, Offset(args.offset))
        pumps.append(pump(pid, receiver, deadline))
    client.run_daemon()
    loop.run_until_complete(asyncio.gather(*pumps))
    client.stop()
    loop.close()
except KeyboardInterrupt:
    pass
            # simulate an async event processing
            await asyncio.sleep(0.05)
        except asyncio.TimeoutError:
            logger.info("No events received, queue size %d, delivered %d", recv.messages.qsize(), recv.delivered)

try:
    ADDRESS = ("amqps://"
               "<URL-encoded-SAS-policy>"
               ":"
               "<URL-encoded-SAS-key>"
               "@"
               "<mynamespace>.servicebus.windows.net"
               "/"
               "myeventhub")
    CONSUMER_GROUP = "$default"
    OFFSET = Offset("-1")

    logger.info("starting loop")
    loop = asyncio.get_event_loop()
    receiver = AsyncReceiver()
    client = EventHubClient(ADDRESS if len(sys.argv) == 1 else sys.argv[1]) \
        .subscribe(receiver, CONSUMER_GROUP, "0", OFFSET) \
        .run_daemon()

    loop.run_until_complete(pump(receiver, 1000))
    client.stop()
    loop.close()

except KeyboardInterrupt:
    pass