def test_exception_raised_by_timeout(self):
herald = DeliveryHerald()
notified_ok = True
def fn():
try:
sleep(1.5)
herald.notify_delivered(MessageId(halink_ctx=43, tag=3))
except:
logging.exception('*** ERROR ***')
notified_ok = False
t = Thread(target=fn)
t.start()
m = MessageId
try:
with self.assertRaises(NotDelivered):
herald.wait_for_any(HaLinkMessagePromise(
[m(42, 1), m(42, 3), m(42, 4)]),
timeout_sec=5)
finally:
t.join()
self.assertTrue(notified_ok,
'Unexpected exception appeared in notifier thread')
def test_works_under_load(self):
herald = DeliveryHerald()
notified_ok = True
def fn(msg: MessageId):
try:
sleep(1.5)
herald.notify_delivered(msg)
except:
logging.exception('*** ERROR ***')
notified_ok = False
threads = [
Thread(target=fn, args=(MessageId(100, i), ))
for i in range(1, 32)
]
for t in threads:
t.start()
def m(x):
return MessageId(halink_ctx=100, tag=x)
try:
herald.wait_for_all(HaLinkMessagePromise(
[m(5), m(25), m(28), m(31)]),
timeout_sec=5)
finally:
for t in threads:
t.join()
self.assertTrue(notified_ok,
'Unexpected exception appeared in notifier thread')
def test_works_if_all_messages_confirmed(self):
herald = DeliveryHerald()
notified_ok = True
def fn():
try:
sleep(1.5)
herald.notify_delivered(MessageId(halink_ctx=42, tag=3))
herald.notify_delivered(MessageId(halink_ctx=42, tag=1))
except:
logging.exception('*** ERROR ***')
notified_ok = False
t = Thread(target=fn)
t.start()
m = MessageId
try:
herald.wait_for_all(HaLinkMessagePromise([m(42, 1),
m(42, 3)]),
timeout_sec=5)
finally:
t.join()
self.assertTrue(notified_ok,
'Unexpected exception appeared in notifier thread')
def main():
# Note: no logging must happen before this call.
# Otherwise the log configuration will not apply.
setup_logging()
# [KN] The elements in the queue will appear if
# 1. A callback is invoked from ha_link (this will happen in a motr
# thread which must be free ASAP)
# 2. A new HA notification has come form Consul via HTTP
# [KN] The messages are consumed by Python thread created by
# _run_qconsumer_thread function.
#
# [KN] Note: The server is launched in the main thread.
planner = WorkPlanner()
util: ConsulUtil = ConsulUtil()
_remove_stale_session(util)
cfg: HL_Fids = _get_motr_fids(util)
LOG.info('Welcome to HaX')
LOG.info(f'Setting up ha_link interface with the options as follows: '
f'hax fid = {cfg.hax_fid}, hax endpoint = {cfg.hax_ep}, '
f'HA fid = {cfg.ha_fid}')
ffi = HaxFFI()
herald = DeliveryHerald()
motr = Motr(planner=planner, ffi=ffi, herald=herald, consul_util=util)
# Note that consumer thread must be started before we invoke motr.start(..)
# Reason: hax process will send entrypoint request and somebody needs
# to reply it.
# TODO make the number of threads configurable
consumer_threads = [
_run_qconsumer_thread(planner, motr, herald, util, i) for i in range(4)
]
try:
# [KN] We use just the first profile for Spiel API for now.
motr.start(cfg.hax_ep,
process=cfg.hax_fid,
ha_service=cfg.ha_fid,
profile=cfg.profiles[0])
LOG.info('Motr API has been started')
rconfc_starter = _run_rconfc_starter_thread(motr, consul_util=util)
stats_updater = _run_stats_updater_thread(motr, consul_util=util)
event_poller = _run_thread(create_ha_thread(planner, util))
# [KN] This is a blocking call. It will work until the program is
# terminated by signal
server = ServerRunner(planner, herald, consul_util=util)
server.run(threads_to_wait=[
*consumer_threads, stats_updater, rconfc_starter, event_poller
])
except Exception:
LOG.exception('Exiting due to an exception')
finally:
motr.fini()
def main():
# Note: no logging must happen before this call.
# Otherwise the log configuration will not apply.
_setup_logging()
# [KN] The elements in the queue will appear if
# 1. A callback is invoked from ha_link (this will happen in a motr
# thread which must be free ASAP)
# 2. A new HA notification has come form Consul via HTTP
# [KN] The messages are consumed by Python thread created by
# _run_qconsumer_thread function.
#
# [KN] Note: The server is launched in the main thread.
q: Queue = Queue(maxsize=8)
util: ConsulUtil = ConsulUtil()
cfg = _get_motr_fids(util)
LOG.info('Welcome to HaX')
LOG.info(f'Setting up ha_link interface with the options as follows: '
f'hax fid = {cfg.hax_fid}, hax endpoint = {cfg.hax_ep}, '
f'HA fid = {cfg.ha_fid}, RM fid = {cfg.rm_fid}')
ffi = HaxFFI()
herald = DeliveryHerald()
motr = Motr(queue=q,
rm_fid=cfg.rm_fid,
ffi=ffi,
herald=herald,
consul_util=util)
# Note that consumer thread must be started before we invoke motr.start(..)
# Reason: hax process will send entrypoint request and somebody needs
# to reply it.
consumer = _run_qconsumer_thread(q, motr, herald)
try:
motr.start(cfg.hax_ep,
process=cfg.hax_fid,
ha_service=cfg.ha_fid,
rm_service=cfg.rm_fid)
LOG.info('Motr API has been started')
stats_updater = _run_stats_updater_thread(motr, consul_util=util)
# [KN] This is a blocking call. It will work until the program is
# terminated by signal
run_server(q,
herald,
consul_util=util,
threads_to_wait=[consumer, stats_updater])
except Exception:
LOG.exception('Exiting due to an exception')
finally:
motr.close()
def test_if_delivered_earlier_than_awaited_notified_immediately(self):
herald = DeliveryHerald()
notified_ok = True
thread_count = 1
latch = CountDownLatch(thread_count)
def fn(msg: MessageId):
try:
LOG.debug('Thread started')
herald.notify_delivered(msg)
LOG.debug('Notified delivery %s', msg)
latch.count_down()
LOG.debug('Main thread unblocked')
except:
logging.exception('*** ERROR ***')
notified_ok = False
threads = [
Thread(target=fn, args=(MessageId(100, i + 1), ))
for i in range(thread_count)
]
for t in threads:
t.start()
# Block until all the threads come to latch.count_down() and thus
# the message is notified for sure
latch.await()
def m(x):
return MessageId(halink_ctx=100, tag=x)
try:
started = time()
herald.wait_for_all(HaLinkMessagePromise([m(1)]),
timeout_sec=2)
finished = time()
finally:
for t in threads:
t.join()
self.assertTrue(notified_ok,
'Unexpected exception appeared in notifier thread')
self.assertLess(
finished - started, 5,
'Awaiting thread was unblocked only by a timeout. It means '
'that unsorted_deliveries was analyzed too late.'
)
def test_if_delivered_earlier_than_awaited_wait_many(self):
herald = DeliveryHerald()
notified_ok = True
thread_count = 6
latch = CountDownLatch(thread_count)
def fn(msg: MessageId):
try:
LOG.debug('Thread started')
herald.notify_delivered(msg)
LOG.debug('Notified delivery %s', msg)
latch.count_down()
LOG.debug('Main thread unblocked')
except:
logging.exception('*** ERROR ***')
notified_ok = False
threads = [
Thread(target=fn, args=(MessageId(100, i + 1), ))
for i in range(thread_count)
]
for t in threads:
t.start()
# Block until all the threads come to latch.count_down() and thus
# the message is notified for sure
latch.await()
def m(x):
return MessageId(halink_ctx=100, tag=x)
try:
herald.wait_for_all(HaLinkMessagePromise([m(1), m(5)]),
timeout_sec=2)
finally:
for t in threads:
t.join()
self.assertTrue(notified_ok,
'Unexpected exception appeared in notifier thread')
self.assertEqual(4, len(herald.unsorted_deliveries.keys()))
def test_it_works(self):
herald = DeliveryHerald()
notified_ok = True
def fn():
try:
sleep(1.5)
herald.notify_delivered(MessageId(halink_ctx=100, tag=1))
except:
logging.exception('*** ERROR ***')
notified_ok = False
t = Thread(target=fn)
t.start()
m = MessageId
herald.wait_for_any(HaLinkMessagePromise(
[m(100, 1), m(100, 3), m(100, 4)]),
timeout_sec=10)
t.join()
self.assertTrue(notified_ok,
'Unexpected exception appeared in notifier thread')
def herald(mocker):
return DeliveryHerald()
def herald(mocker):
herald = DeliveryHerald()
mocker.patch.object(herald, 'wait_for_all')
return herald
def main():
# Note: no logging must happen before this call.
# Otherwise the log configuration will not apply.
setup_logging()
set_locale()
inject.configure(di_configuration)
state = inject.instance(HaxGlobalState)
# [KN] The elements in the work planner will appear if
# 1. A callback is invoked from ha_link (this will happen in a motr
# thread which must be free ASAP)
# 2. A new HA notification has come form Consul via HTTP
# [KN] The messages are consumed by Python threads created by
# _run_qconsumer_thread function.
#
# [KN] Note: The server is launched in the main thread.
planner = WorkPlanner()
def handle_signal(sig, frame):
state.set_stopping()
planner.shutdown()
# This is necessary to allow hax to exit early if Consul is not available
# (otherwise _get_motr_fids() may be retrying forever even if the hax
# process needs to shutdown).
signal.signal(signal.SIGINT, handle_signal)
util: ConsulUtil = ConsulUtil()
# Avoid removing session on hax start as this will happen
# on every node, thus leader election will keep re-triggering
# until the final hax node starts, this will delay further
# bootstrapping operations.
_remove_stale_session(util)
cfg: HL_Fids = _get_motr_fids(util)
hax_http_port = util.get_hax_http_port()
util.init_motr_processes_status()
LOG.info('Welcome to HaX')
LOG.info(f'Setting up ha_link interface with the options as follows: '
f'hax fid = {cfg.hax_fid}, hax endpoint = {cfg.hax_ep}, '
f'HA fid = {cfg.ha_fid}')
ffi = HaxFFI()
herald = DeliveryHerald()
motr = Motr(planner=planner, ffi=ffi, herald=herald, consul_util=util)
# Note that consumer thread must be started before we invoke motr.start(..)
# Reason: hax process will send entrypoint request and somebody needs
# to reply it.
# TODO make the number of threads configurable
consumer_threads = [
_run_qconsumer_thread(planner, motr, herald, util, i)
for i in range(32)
]
try:
# [KN] We use just the first profile for Spiel API for now.
motr.start(cfg.hax_ep,
process=cfg.hax_fid,
ha_service=cfg.ha_fid,
profile=cfg.profiles[0])
LOG.info('Motr API has been started')
rconfc_starter = _run_rconfc_starter_thread(motr, consul_util=util)
stats_updater = _run_stats_updater_thread(motr, consul_util=util)
bc_updater = _run_bc_updater_thread(motr, consul_util=util)
event_poller = _run_thread(create_ha_thread(planner, util))
# [KN] This is a blocking call. It will work until the program is
# terminated by signal
server = ServerRunner(planner,
herald,
consul_util=util,
hax_state=state)
server.run(threads_to_wait=[
*consumer_threads, stats_updater, bc_updater, rconfc_starter,
event_poller
],
port=hax_http_port)
except Exception:
LOG.exception('Exiting due to an exception')
finally:
motr.fini()