コード例 #1
0
    def test_exception_raised_by_timeout(self):
        herald = DeliveryHerald()
        notified_ok = True

        def fn():
            try:
                sleep(1.5)
                herald.notify_delivered(MessageId(halink_ctx=43, tag=3))
            except:
                logging.exception('*** ERROR ***')
                notified_ok = False

        t = Thread(target=fn)
        t.start()

        m = MessageId
        try:
            with self.assertRaises(NotDelivered):
                herald.wait_for_any(HaLinkMessagePromise(
                    [m(42, 1), m(42, 3), m(42, 4)]),
                                    timeout_sec=5)
        finally:
            t.join()
        self.assertTrue(notified_ok,
                        'Unexpected exception appeared in notifier thread')
コード例 #2
0
    def test_works_under_load(self):
        herald = DeliveryHerald()
        notified_ok = True

        def fn(msg: MessageId):
            try:
                sleep(1.5)
                herald.notify_delivered(msg)
            except:
                logging.exception('*** ERROR ***')
                notified_ok = False

        threads = [
            Thread(target=fn, args=(MessageId(100, i), ))
            for i in range(1, 32)
        ]
        for t in threads:
            t.start()

        def m(x):
            return MessageId(halink_ctx=100, tag=x)

        try:
            herald.wait_for_all(HaLinkMessagePromise(
                [m(5), m(25), m(28), m(31)]),
                                timeout_sec=5)
        finally:
            for t in threads:
                t.join()
        self.assertTrue(notified_ok,
                        'Unexpected exception appeared in notifier thread')
コード例 #3
0
    def test_works_if_all_messages_confirmed(self):
        herald = DeliveryHerald()
        notified_ok = True

        def fn():
            try:
                sleep(1.5)
                herald.notify_delivered(MessageId(halink_ctx=42, tag=3))
                herald.notify_delivered(MessageId(halink_ctx=42, tag=1))
            except:
                logging.exception('*** ERROR ***')
                notified_ok = False

        t = Thread(target=fn)
        t.start()

        m = MessageId
        try:
            herald.wait_for_all(HaLinkMessagePromise([m(42, 1),
                                                      m(42, 3)]),
                                timeout_sec=5)
        finally:
            t.join()
        self.assertTrue(notified_ok,
                        'Unexpected exception appeared in notifier thread')
コード例 #4
0
    def test_if_delivered_earlier_than_awaited_notified_immediately(self):
        herald = DeliveryHerald()
        notified_ok = True
        thread_count = 1
        latch = CountDownLatch(thread_count)

        def fn(msg: MessageId):
            try:
                LOG.debug('Thread started')
                herald.notify_delivered(msg)
                LOG.debug('Notified delivery %s', msg)
                latch.count_down()
                LOG.debug('Main thread unblocked')

            except:
                logging.exception('*** ERROR ***')
                notified_ok = False

        threads = [
            Thread(target=fn, args=(MessageId(100, i + 1), ))
            for i in range(thread_count)
        ]

        for t in threads:
            t.start()
        # Block until all the threads come to latch.count_down() and thus
        # the message is notified for sure
        latch.await()

        def m(x):
            return MessageId(halink_ctx=100, tag=x)

        try:
            started = time()
            herald.wait_for_all(HaLinkMessagePromise([m(1)]),
                                timeout_sec=2)
            finished = time()
        finally:
            for t in threads:
                t.join()
        self.assertTrue(notified_ok,
                        'Unexpected exception appeared in notifier thread')
        self.assertLess(
            finished - started, 5,
            'Awaiting thread was unblocked only by a timeout. It means '
            'that unsorted_deliveries was analyzed too late.'
        )
コード例 #5
0
def main():
    # Note: no logging must happen before this call.
    # Otherwise the log configuration will not apply.
    setup_logging()

    # [KN] The elements in the queue will appear if
    # 1. A callback is invoked from ha_link (this will happen in a motr
    #    thread which must be free ASAP)
    # 2. A new HA notification has come form Consul via HTTP
    # [KN] The messages are consumed by Python thread created by
    # _run_qconsumer_thread function.
    #
    # [KN] Note: The server is launched in the main thread.
    planner = WorkPlanner()

    util: ConsulUtil = ConsulUtil()
    _remove_stale_session(util)
    cfg: HL_Fids = _get_motr_fids(util)

    LOG.info('Welcome to HaX')
    LOG.info(f'Setting up ha_link interface with the options as follows: '
             f'hax fid = {cfg.hax_fid}, hax endpoint = {cfg.hax_ep}, '
             f'HA fid = {cfg.ha_fid}')

    ffi = HaxFFI()
    herald = DeliveryHerald()
    motr = Motr(planner=planner, ffi=ffi, herald=herald, consul_util=util)

    # Note that consumer thread must be started before we invoke motr.start(..)
    # Reason: hax process will send entrypoint request and somebody needs
    # to reply it.

    # TODO make the number of threads configurable
    consumer_threads = [
        _run_qconsumer_thread(planner, motr, herald, util, i) for i in range(4)
    ]

    try:
        # [KN] We use just the first profile for Spiel API for now.
        motr.start(cfg.hax_ep,
                   process=cfg.hax_fid,
                   ha_service=cfg.ha_fid,
                   profile=cfg.profiles[0])
        LOG.info('Motr API has been started')
        rconfc_starter = _run_rconfc_starter_thread(motr, consul_util=util)

        stats_updater = _run_stats_updater_thread(motr, consul_util=util)
        event_poller = _run_thread(create_ha_thread(planner, util))
        # [KN] This is a blocking call. It will work until the program is
        # terminated by signal

        server = ServerRunner(planner, herald, consul_util=util)
        server.run(threads_to_wait=[
            *consumer_threads, stats_updater, rconfc_starter, event_poller
        ])
    except Exception:
        LOG.exception('Exiting due to an exception')
    finally:
        motr.fini()
コード例 #6
0
    def test_if_delivered_earlier_than_awaited_wait_many(self):
        herald = DeliveryHerald()
        notified_ok = True
        thread_count = 6
        latch = CountDownLatch(thread_count)

        def fn(msg: MessageId):
            try:
                LOG.debug('Thread started')
                herald.notify_delivered(msg)
                LOG.debug('Notified delivery %s', msg)
                latch.count_down()
                LOG.debug('Main thread unblocked')

            except:
                logging.exception('*** ERROR ***')
                notified_ok = False

        threads = [
            Thread(target=fn, args=(MessageId(100, i + 1), ))
            for i in range(thread_count)
        ]

        for t in threads:
            t.start()
        # Block until all the threads come to latch.count_down() and thus
        # the message is notified for sure
        latch.await()

        def m(x):
            return MessageId(halink_ctx=100, tag=x)

        try:
            herald.wait_for_all(HaLinkMessagePromise([m(1), m(5)]),
                                timeout_sec=2)
        finally:
            for t in threads:
                t.join()
        self.assertTrue(notified_ok,
                        'Unexpected exception appeared in notifier thread')
        self.assertEqual(4, len(herald.unsorted_deliveries.keys()))
コード例 #7
0
def main():
    # Note: no logging must happen before this call.
    # Otherwise the log configuration will not apply.
    _setup_logging()

    # [KN] The elements in the queue will appear if
    # 1. A callback is invoked from ha_link (this will happen in a motr
    #    thread which must be free ASAP)
    # 2. A new HA notification has come form Consul via HTTP
    # [KN] The messages are consumed by Python thread created by
    # _run_qconsumer_thread function.
    #
    # [KN] Note: The server is launched in the main thread.
    q: Queue = Queue(maxsize=8)

    util: ConsulUtil = ConsulUtil()
    cfg = _get_motr_fids(util)

    LOG.info('Welcome to HaX')
    LOG.info(f'Setting up ha_link interface with the options as follows: '
             f'hax fid = {cfg.hax_fid}, hax endpoint = {cfg.hax_ep}, '
             f'HA fid = {cfg.ha_fid}, RM fid = {cfg.rm_fid}')

    ffi = HaxFFI()
    herald = DeliveryHerald()
    motr = Motr(queue=q,
                rm_fid=cfg.rm_fid,
                ffi=ffi,
                herald=herald,
                consul_util=util)

    # Note that consumer thread must be started before we invoke motr.start(..)
    # Reason: hax process will send entrypoint request and somebody needs
    # to reply it.
    consumer = _run_qconsumer_thread(q, motr, herald)

    try:
        motr.start(cfg.hax_ep,
                   process=cfg.hax_fid,
                   ha_service=cfg.ha_fid,
                   rm_service=cfg.rm_fid)
        LOG.info('Motr API has been started')
        stats_updater = _run_stats_updater_thread(motr, consul_util=util)
        # [KN] This is a blocking call. It will work until the program is
        # terminated by signal
        run_server(q,
                   herald,
                   consul_util=util,
                   threads_to_wait=[consumer, stats_updater])
    except Exception:
        LOG.exception('Exiting due to an exception')
    finally:
        motr.close()
コード例 #8
0
    def test_it_works(self):
        herald = DeliveryHerald()
        notified_ok = True

        def fn():
            try:
                sleep(1.5)
                herald.notify_delivered(MessageId(halink_ctx=100, tag=1))
            except:
                logging.exception('*** ERROR ***')
                notified_ok = False

        t = Thread(target=fn)
        t.start()

        m = MessageId
        herald.wait_for_any(HaLinkMessagePromise(
            [m(100, 1), m(100, 3), m(100, 4)]),
                            timeout_sec=10)
        t.join()
        self.assertTrue(notified_ok,
                        'Unexpected exception appeared in notifier thread')
コード例 #9
0
ファイル: test_server.py プロジェクト: madhavemuri/cortx-hare
def herald(mocker):
    return DeliveryHerald()
コード例 #10
0
def herald(mocker):
    herald = DeliveryHerald()
    mocker.patch.object(herald, 'wait_for_all')
    return herald
コード例 #11
0
def main():
    # Note: no logging must happen before this call.
    # Otherwise the log configuration will not apply.
    setup_logging()
    set_locale()
    inject.configure(di_configuration)

    state = inject.instance(HaxGlobalState)

    # [KN] The elements in the work planner will appear if
    # 1. A callback is invoked from ha_link (this will happen in a motr
    #    thread which must be free ASAP)
    # 2. A new HA notification has come form Consul via HTTP
    # [KN] The messages are consumed by Python threads created by
    # _run_qconsumer_thread function.
    #
    # [KN] Note: The server is launched in the main thread.
    planner = WorkPlanner()

    def handle_signal(sig, frame):
        state.set_stopping()
        planner.shutdown()

    # This is necessary to allow hax to exit early if Consul is not available
    # (otherwise _get_motr_fids() may be retrying forever even if the hax
    # process needs to shutdown).
    signal.signal(signal.SIGINT, handle_signal)

    util: ConsulUtil = ConsulUtil()
    # Avoid removing session on hax start as this will happen
    # on every node, thus leader election will keep re-triggering
    # until the final hax node starts, this will delay further
    # bootstrapping operations.
    _remove_stale_session(util)
    cfg: HL_Fids = _get_motr_fids(util)
    hax_http_port = util.get_hax_http_port()
    util.init_motr_processes_status()

    LOG.info('Welcome to HaX')
    LOG.info(f'Setting up ha_link interface with the options as follows: '
             f'hax fid = {cfg.hax_fid}, hax endpoint = {cfg.hax_ep}, '
             f'HA fid = {cfg.ha_fid}')

    ffi = HaxFFI()
    herald = DeliveryHerald()
    motr = Motr(planner=planner, ffi=ffi, herald=herald, consul_util=util)

    # Note that consumer thread must be started before we invoke motr.start(..)
    # Reason: hax process will send entrypoint request and somebody needs
    # to reply it.

    # TODO make the number of threads configurable
    consumer_threads = [
        _run_qconsumer_thread(planner, motr, herald, util, i)
        for i in range(32)
    ]

    try:
        # [KN] We use just the first profile for Spiel API for now.
        motr.start(cfg.hax_ep,
                   process=cfg.hax_fid,
                   ha_service=cfg.ha_fid,
                   profile=cfg.profiles[0])
        LOG.info('Motr API has been started')
        rconfc_starter = _run_rconfc_starter_thread(motr, consul_util=util)

        stats_updater = _run_stats_updater_thread(motr, consul_util=util)
        bc_updater = _run_bc_updater_thread(motr, consul_util=util)
        event_poller = _run_thread(create_ha_thread(planner, util))
        # [KN] This is a blocking call. It will work until the program is
        # terminated by signal

        server = ServerRunner(planner,
                              herald,
                              consul_util=util,
                              hax_state=state)
        server.run(threads_to_wait=[
            *consumer_threads, stats_updater, bc_updater, rconfc_starter,
            event_poller
        ],
                   port=hax_http_port)
    except Exception:
        LOG.exception('Exiting due to an exception')
    finally:
        motr.fini()