def test_entrypoint_requests_share_same_group(self):
planner = WorkPlanner()
ep1 = entrypoint()
ep2 = entrypoint()
ep1 = planner._assign_group(ep1)
ep2 = planner._assign_group(ep2)
self.assertEqual([0, 0], [ep1.group, ep2.group])
def test_entrypoint_not_paralleled_with_broadcast(self):
planner = WorkPlanner()
bcast = broadcast()
ep1 = entrypoint()
bcast = planner._assign_group(bcast)
ep1 = planner._assign_group(ep1)
self.assertEqual([0, 1], [bcast.group, ep1.group])
def test_no_hang_when_group_id_cycled(self):
planner = WorkPlanner()
def my_state():
return State(next_group_id=99999,
active_commands=LinkedList(),
current_group_id=99999,
next_group_commands=set(),
is_shutdown=False)
planner = WorkPlanner(init_state_factory=my_state)
tracker = GroupTracker()
thread_count = 1
for i in range(10):
planner.add_command(process_event())
for j in range(thread_count):
planner.add_command(Die())
exc = None
def fn(planner: WorkPlanner):
nonlocal exc
try:
while True:
LOG.log(TRACE, "Requesting for a work")
# import pudb.remote
# pudb.remote.set_trace(term_size=(120, 40), port=9998)
cmd = planner.get_next_command()
LOG.log(TRACE, "The command is received %s [group=%s]",
type(cmd), cmd.group)
if isinstance(cmd, Die):
LOG.log(TRACE,
"Poison pill is received - exiting. Bye!")
planner.notify_finished(cmd)
break
tracker.log(cmd)
LOG.log(TRACE, "The job is done, notifying the planner")
planner.notify_finished(cmd)
LOG.log(TRACE, "Notified. ")
except Exception as e:
LOG.exception('*** ERROR ***')
exc = e
workers = [
Thread(target=fn, args=(planner, )) for t in range(thread_count)
]
for t in workers:
t.start()
for t in workers:
t.join()
if exc:
raise exc
groups_processed = tracker.get_tracks()
self.assertEqual([99999, 10**5, 0, 1, 2, 3, 4,
5, 6, 7], groups_processed)
def test_ha_nvec_get_shares_group_always(self):
planner = WorkPlanner()
assign = planner._assign_group
msgs_after_bc = [
assign(broadcast()),
assign(nvec_get()),
assign(broadcast()),
assign(entrypoint())
]
msgs_after_ep = [
assign(entrypoint()),
assign(nvec_get()),
assign(broadcast()),
assign(entrypoint())
]
msgs_after_nvec = [
assign(entrypoint()),
assign(nvec_get()),
assign(nvec_get()),
assign(entrypoint())
]
self.assertEqual([0, 0, 1, 2], [m.group for m in msgs_after_bc])
self.assertEqual([2, 2, 3, 4], [m.group for m in msgs_after_ep])
self.assertEqual([4, 4, 4, 4], [m.group for m in msgs_after_nvec])
def main():
# Note: no logging must happen before this call.
# Otherwise the log configuration will not apply.
setup_logging()
# [KN] The elements in the queue will appear if
# 1. A callback is invoked from ha_link (this will happen in a motr
# thread which must be free ASAP)
# 2. A new HA notification has come form Consul via HTTP
# [KN] The messages are consumed by Python thread created by
# _run_qconsumer_thread function.
#
# [KN] Note: The server is launched in the main thread.
planner = WorkPlanner()
util: ConsulUtil = ConsulUtil()
_remove_stale_session(util)
cfg: HL_Fids = _get_motr_fids(util)
LOG.info('Welcome to HaX')
LOG.info(f'Setting up ha_link interface with the options as follows: '
f'hax fid = {cfg.hax_fid}, hax endpoint = {cfg.hax_ep}, '
f'HA fid = {cfg.ha_fid}')
ffi = HaxFFI()
herald = DeliveryHerald()
motr = Motr(planner=planner, ffi=ffi, herald=herald, consul_util=util)
# Note that consumer thread must be started before we invoke motr.start(..)
# Reason: hax process will send entrypoint request and somebody needs
# to reply it.
# TODO make the number of threads configurable
consumer_threads = [
_run_qconsumer_thread(planner, motr, herald, util, i) for i in range(4)
]
try:
# [KN] We use just the first profile for Spiel API for now.
motr.start(cfg.hax_ep,
process=cfg.hax_fid,
ha_service=cfg.ha_fid,
profile=cfg.profiles[0])
LOG.info('Motr API has been started')
rconfc_starter = _run_rconfc_starter_thread(motr, consul_util=util)
stats_updater = _run_stats_updater_thread(motr, consul_util=util)
event_poller = _run_thread(create_ha_thread(planner, util))
# [KN] This is a blocking call. It will work until the program is
# terminated by signal
server = ServerRunner(planner, herald, consul_util=util)
server.run(threads_to_wait=[
*consumer_threads, stats_updater, rconfc_starter, event_poller
])
except Exception:
LOG.exception('Exiting due to an exception')
finally:
motr.fini()
def fn(planner: WorkPlanner):
nonlocal exc
try:
while True:
LOG.log(TRACE, "Requesting for a work")
cmd = planner.get_next_command()
LOG.log(TRACE, "The command is received %s [group=%s]",
type(cmd), cmd.group)
if isinstance(cmd, BroadcastHAStates):
time.sleep(1)
if isinstance(cmd, EntrypointRequest):
planner.shutdown()
if isinstance(cmd, Die):
LOG.log(TRACE,
"Poison pill is received - exiting. Bye!")
planner.notify_finished(cmd)
break
tracker.log(cmd)
LOG.log(TRACE, "The job is done, notifying the planner")
planner.notify_finished(cmd)
LOG.log(TRACE, "Notified. ")
except Exception as e:
LOG.exception('*** ERROR ***')
exc = e
def test_entrypoint_executed_asap(self):
planner = WorkPlanner()
a = planner._assign_group
self.assertEqual(
[0, 1, 0],
[a(i)[0].group
for i in [broadcast(), broadcast(),
entrypoint()]])
def fn(planner: WorkPlanner):
nonlocal exc
try:
while True:
LOG.log(TRACE, "Requesting for a work")
cmd = planner.get_next_command()
LOG.log(TRACE, "The command is received")
if isinstance(cmd, Die):
LOG.log(TRACE,
"Poison pill is received - exiting. Bye!")
break
sleep(0.5)
LOG.log(TRACE, "The job is done, notifying the planner")
planner.notify_finished(cmd)
LOG.log(TRACE, "Notified. ")
except Exception as e:
LOG.exception('*** ERROR ***')
exc = e
def test_broadcast_starts_new_group(self):
planner = WorkPlanner()
assign = planner._assign_group
msgs = [
assign(broadcast()),
assign(broadcast()),
assign(broadcast()),
assign(entrypoint())
]
self.assertEqual([0, 1, 2, 3], [m.group for m in msgs])
def test_broadcast_does_not_start_new_group(self):
planner = WorkPlanner()
assign = planner._assign_group
msgs = [
assign(broadcast()),
assign(broadcast()),
assign(broadcast()),
assign(nvec_get())
]
self.assertEqual([0, 1, 2, 0], [m.group for (m, _) in msgs])
def fn(planner: WorkPlanner):
nonlocal exc
try:
while True:
LOG.log(TRACE, "Requesting for a work")
# import pudb.remote
# pudb.remote.set_trace(term_size=(120, 40), port=9998)
cmd = planner.get_next_command()
LOG.log(TRACE, "The command is received %s [group=%s]",
type(cmd), cmd.group)
if isinstance(cmd, Die):
LOG.log(TRACE,
"Poison pill is received - exiting. Bye!")
planner.notify_finished(cmd)
break
tracker.log(cmd)
LOG.log(TRACE, "The job is done, notifying the planner")
planner.notify_finished(cmd)
LOG.log(TRACE, "Notified. ")
except Exception as e:
LOG.exception('*** ERROR ***')
exc = e
def test_group_id_cycled(self):
def my_state():
return State(next_group_id=99999,
active_commands=LinkedList(),
current_group_id=99999,
next_group_commands=set(),
is_shutdown=False)
planner = WorkPlanner(init_state_factory=my_state)
assign = planner._assign_group
msgs = [
assign(broadcast()),
assign(broadcast()),
assign(broadcast()),
assign(broadcast()),
]
self.assertEqual([99999, 10**5, 0, 1], [m.group for (m, _) in msgs])
def test_parallelism_is_possible(self):
planner = WorkPlanner()
for i in range(40):
planner.add_command(entrypoint())
for j in range(4):
planner.add_command(Die())
exc = None
def fn(planner: WorkPlanner):
nonlocal exc
try:
while True:
LOG.log(TRACE, "Requesting for a work")
cmd = planner.get_next_command()
LOG.log(TRACE, "The command is received")
if isinstance(cmd, Die):
LOG.log(TRACE,
"Poison pill is received - exiting. Bye!")
break
sleep(0.5)
LOG.log(TRACE, "The job is done, notifying the planner")
planner.notify_finished(cmd)
LOG.log(TRACE, "Notified. ")
except Exception as e:
LOG.exception('*** ERROR ***')
exc = e
workers = [Thread(target=fn, args=(planner, )) for t in range(4)]
time_1 = time.time()
for t in workers:
t.start()
for t in workers:
t.join()
time_2 = time.time()
logging.info('Processing time %s', time_2 - time_1)
if exc:
raise exc
self.assertTrue(planner.is_empty(), 'Not all commands were read out')
# Every thread sleeps for 500ms. 40 commands * 0.5 gives 20 seconds if
# the commands executed sequentially
self.assertLess(time_2 - time_1, 19, 'Suspiciously slow')
def update_process_failure(self, planner: WorkPlanner,
ha_states: List[HAState]) -> List[HAState]:
new_ha_states: List[HAState] = []
for state in ha_states:
# We are only concerned with process statuses.
if state.fid.container == ObjT.PROCESS.value:
current_status = self.consul.get_process_current_status(
state.status, state.fid)
if current_status == ServiceHealth.OK:
if (self.consul.get_process_local_status(
state.fid) == 'M0_CONF_HA_PROCESS_STARTED'):
continue
if current_status in (ServiceHealth.FAILED,
ServiceHealth.STOPPED):
if (self.consul.get_process_local_status(
state.fid) == 'M0_CONF_HA_PROCESS_STOPPED'):
# Consul may report failure of a process multiple
# times, so we don't want to send duplicate failure
# notifications, it may cause delay in cleanup
# activities.
continue
if current_status == ServiceHealth.UNKNOWN:
# We got service status as UNKNOWN, that means hax was
# notified about process failure but hax couldn't
# confirm if the process is in failed state or have
# failed and restarted. So, we will not loose the
# event and try again to confirm the real time
# process status by enqueing a broadcast event
# specific to this process.
# It is expected that the process status gets
# eventually confirmed as either failed or passing (OK).
# This situation typically arises due to delay
# in receiving failure notification during which the
# corresponding process might be restarting or have
# already restarted. Thus it is important to confirm
# the real time status of the process before
# broadcasting failure.
current_status = ServiceHealth.UNKNOWN
planner.add_command(
BroadcastHAStates(states=[
HAState(fid=state.fid, status=ServiceHealth.FAILED)
],
reply_to=None))
if current_status not in (ServiceHealth.UNKNOWN,
ServiceHealth.OFFLINE):
# We also need to account and report the failure of remote
# Motr processes to this node's hax and motr processes.
# When Consul reports a remote process failure, hax
# confirms its current status from Consul KV and updates
# the list of failed services and also adds it to the
# broadcast list.
if current_status != ServiceHealth.OK:
event = m0HaProcessEvent.M0_CONF_HA_PROCESS_STOPPED
else:
event = m0HaProcessEvent.M0_CONF_HA_PROCESS_STARTED
self.consul.update_process_status(
ConfHaProcess(
chp_event=event,
chp_type=int(
m0HaProcessType.M0_CONF_HA_PROCESS_M0D),
chp_pid=0,
fid=state.fid))
new_ha_states.append(
HAState(fid=state.fid, status=current_status))
else:
new_ha_states.append(state)
return new_ha_states
def _do_work(self, planner: WorkPlanner, motr: Motr):
LOG.info('Handler thread has started')
try:
while True:
try:
LOG.debug('Waiting for the next message')
item = planner.get_next_command()
LOG.debug('Got %s message from planner', item)
if isinstance(item, FirstEntrypointRequest):
motr.send_entrypoint_request_reply(
EntrypointRequest(
reply_context=item.reply_context,
req_id=item.req_id,
remote_rpc_endpoint=item.remote_rpc_endpoint,
process_fid=item.process_fid,
git_rev=item.git_rev,
pid=item.pid,
is_first_request=item.is_first_request))
elif isinstance(item, EntrypointRequest):
# While replying any Exception is catched. In such a
# case, the motr process will receive EAGAIN and
# hence will need to make new attempt by itself
motr.send_entrypoint_request_reply(item)
elif isinstance(item, ProcessEvent):
self._update_process_status(planner, motr, item.evt)
elif isinstance(item, HaNvecGetEvent):
fn = motr.ha_nvec_get_reply
# If a consul-related exception appears, it will
# be processed by repeat_if_fails.
#
# This thread will become blocked until that
# intermittent error gets resolved.
decorated = (repeat_if_fails(wait_seconds=5))(fn)
decorated(item)
elif isinstance(item, HaNvecSetEvent):
fn = motr.ha_nvec_set_process
# If a consul-related exception appears, it will
# be processed by repeat_if_fails.
#
# This thread will become blocked until that
# intermittent error gets resolved.
decorated = (repeat_if_fails(wait_seconds=5))(fn)
decorated(item)
elif isinstance(item, BroadcastHAStates):
LOG.info('HA states: %s', item.states)
ha_states = self.update_process_failure(
planner, item.states)
result: List[MessageId] = motr.broadcast_ha_states(
ha_states)
if item.reply_to:
item.reply_to.put(result)
elif isinstance(item, StobIoqError):
LOG.info('Stob IOQ: %s', item.fid)
payload = dump_json(item)
LOG.debug('Stob IOQ JSON: %s', payload)
offset = self.eq_publisher.publish('stob-ioq', payload)
LOG.debug('Written to epoch: %s', offset)
elif isinstance(item, SnsRepairStatus):
LOG.info('Requesting SNS repair status')
status = motr.get_repair_status(item.fid)
LOG.info('SNS repair status is received: %s', status)
item.reply_to.put(status)
elif isinstance(item, SnsRebalanceStatus):
LOG.info('Requesting SNS rebalance status')
status = motr.get_rebalance_status(item.fid)
LOG.info('SNS rebalance status is received: %s',
status)
item.reply_to.put(status)
elif isinstance(item, SnsRebalanceStart):
LOG.info('Requesting SNS rebalance start')
motr.start_rebalance(item.fid)
elif isinstance(item, SnsRebalanceStop):
LOG.info('Requesting SNS rebalance stop')
motr.stop_rebalance(item.fid)
elif isinstance(item, SnsRebalancePause):
LOG.info('Requesting SNS rebalance pause')
motr.pause_rebalance(item.fid)
elif isinstance(item, SnsRebalanceResume):
LOG.info('Requesting SNS rebalance resume')
motr.resume_rebalance(item.fid)
elif isinstance(item, SnsRepairStart):
LOG.info('Requesting SNS repair start')
motr.start_repair(item.fid)
elif isinstance(item, SnsRepairStop):
LOG.info('Requesting SNS repair stop')
motr.stop_repair(item.fid)
elif isinstance(item, SnsRepairPause):
LOG.info('Requesting SNS repair pause')
motr.pause_repair(item.fid)
elif isinstance(item, SnsRepairResume):
LOG.info('Requesting SNS repair resume')
motr.resume_repair(item.fid)
elif isinstance(item, Die):
raise StopIteration()
else:
LOG.warning('Unsupported event type received: %s',
item)
except StopIteration:
raise
except Exception:
# no op, swallow the exception
LOG.exception('**ERROR**')
finally:
planner.notify_finished(item)
except StopIteration:
LOG.info('Consumer Stopped')
if self.idx == 0:
motr.stop()
finally:
LOG.info('Handler thread has exited')
def main():
# Note: no logging must happen before this call.
# Otherwise the log configuration will not apply.
setup_logging()
set_locale()
inject.configure(di_configuration)
state = inject.instance(HaxGlobalState)
# [KN] The elements in the work planner will appear if
# 1. A callback is invoked from ha_link (this will happen in a motr
# thread which must be free ASAP)
# 2. A new HA notification has come form Consul via HTTP
# [KN] The messages are consumed by Python threads created by
# _run_qconsumer_thread function.
#
# [KN] Note: The server is launched in the main thread.
planner = WorkPlanner()
def handle_signal(sig, frame):
state.set_stopping()
planner.shutdown()
# This is necessary to allow hax to exit early if Consul is not available
# (otherwise _get_motr_fids() may be retrying forever even if the hax
# process needs to shutdown).
signal.signal(signal.SIGINT, handle_signal)
util: ConsulUtil = ConsulUtil()
# Avoid removing session on hax start as this will happen
# on every node, thus leader election will keep re-triggering
# until the final hax node starts, this will delay further
# bootstrapping operations.
_remove_stale_session(util)
cfg: HL_Fids = _get_motr_fids(util)
hax_http_port = util.get_hax_http_port()
util.init_motr_processes_status()
LOG.info('Welcome to HaX')
LOG.info(f'Setting up ha_link interface with the options as follows: '
f'hax fid = {cfg.hax_fid}, hax endpoint = {cfg.hax_ep}, '
f'HA fid = {cfg.ha_fid}')
ffi = HaxFFI()
herald = DeliveryHerald()
motr = Motr(planner=planner, ffi=ffi, herald=herald, consul_util=util)
# Note that consumer thread must be started before we invoke motr.start(..)
# Reason: hax process will send entrypoint request and somebody needs
# to reply it.
# TODO make the number of threads configurable
consumer_threads = [
_run_qconsumer_thread(planner, motr, herald, util, i)
for i in range(32)
]
try:
# [KN] We use just the first profile for Spiel API for now.
motr.start(cfg.hax_ep,
process=cfg.hax_fid,
ha_service=cfg.ha_fid,
profile=cfg.profiles[0])
LOG.info('Motr API has been started')
rconfc_starter = _run_rconfc_starter_thread(motr, consul_util=util)
stats_updater = _run_stats_updater_thread(motr, consul_util=util)
bc_updater = _run_bc_updater_thread(motr, consul_util=util)
event_poller = _run_thread(create_ha_thread(planner, util))
# [KN] This is a blocking call. It will work until the program is
# terminated by signal
server = ServerRunner(planner,
herald,
consul_util=util,
hax_state=state)
server.run(threads_to_wait=[
*consumer_threads, stats_updater, bc_updater, rconfc_starter,
event_poller
],
port=hax_http_port)
except Exception:
LOG.exception('Exiting due to an exception')
finally:
motr.fini()
def test_groups_processed_sequentially_4_threads(self):
planner = WorkPlanner()
group_idx = 0
def ret_values(cmd: BaseMessage) -> bool:
nonlocal group_idx
# We don't care about the group distribution logic
# in this test. Instead, we concentrate how different group
# numbers are processed by the workers and the order
# in which they are allowed to process the messages.
#
# _assign_group is invoked under a lock acquired, so this
# increment is thread-safe.
values = [0, 1, 0, 1, 0, 1, 0, 1, 0, 1]
ret = bool(values[group_idx])
group_idx += 1
return ret
setattr(planner, '_should_increase_group',
Mock(side_effect=ret_values))
tracker = GroupTracker()
thread_count = 4
for i in range(10):
planner.add_command(entrypoint())
for j in range(thread_count):
planner.add_command(Die())
exc = None
def fn(planner: WorkPlanner):
nonlocal exc
try:
while True:
LOG.log(TRACE, "Requesting for a work")
cmd = planner.get_next_command()
LOG.log(TRACE, "The command is received %s [group=%s]",
type(cmd), cmd.group)
if isinstance(cmd, Die):
LOG.log(TRACE,
"Poison pill is received - exiting. Bye!")
planner.notify_finished(cmd)
break
tracker.log(cmd)
LOG.log(TRACE, "The job is done, notifying the planner")
planner.notify_finished(cmd)
LOG.log(TRACE, "Notified. ")
except Exception as e:
LOG.exception('*** ERROR ***')
exc = e
workers = [
Thread(target=fn, args=(planner, )) for t in range(thread_count)
]
for t in workers:
t.start()
for t in workers:
t.join()
if exc:
raise exc
groups_processed = tracker.get_tracks()
self.assertEqual([0, 1, 1, 2, 2, 3, 3, 4, 4, 5], groups_processed)
def test_entrypoint_request_processed_asap(self):
planner = WorkPlanner()
group_idx = 0
def ret_values(cmd: BaseMessage) -> bool:
nonlocal group_idx
# We don't care about the group distribution logic
# in this test. Instead, we concentrate how different group
# numbers are processed by the workers and the order
# in which they are allowed to process the messages.
#
# _assign_group is invoked under a lock acquired, so this
# increment is thread-safe.
values = [0, 1, 0, 1, 0, 1, 0, 1, 0, 1]
ret = bool(values[group_idx])
group_idx += 1
return ret
tracker = TimeTracker()
thread_count = 4
planner.add_command(broadcast())
planner.add_command(broadcast())
planner.add_command(broadcast())
planner.add_command(entrypoint())
for j in range(thread_count):
planner.add_command(Die())
exc = None
def fn(planner: WorkPlanner):
nonlocal exc
try:
while True:
LOG.log(TRACE, "Requesting for a work")
cmd = planner.get_next_command()
LOG.log(TRACE, "The command is received %s [group=%s]",
type(cmd), cmd.group)
if isinstance(cmd, BroadcastHAStates):
time.sleep(1.5)
if isinstance(cmd, Die):
LOG.log(TRACE,
"Poison pill is received - exiting. Bye!")
planner.notify_finished(cmd)
break
tracker.log(cmd)
LOG.log(TRACE, "The job is done, notifying the planner")
planner.notify_finished(cmd)
LOG.log(TRACE, "Notified. ")
except Exception as e:
LOG.exception('*** ERROR ***')
exc = e
workers = [
Thread(target=fn, args=(planner, )) for t in range(thread_count)
]
for t in workers:
t.start()
for t in workers:
t.join()
if exc:
raise exc
tracks = tracker.get_tracks()
(cmd, ts) = tracks[0]
self.assertTrue(isinstance(cmd, EntrypointRequest))
def test_workers_not_blocked_by_future_work(self):
planner = WorkPlanner()
group_idx = 0
tracker = TimeTracker()
thread_count = 2
# We add way more commands than we have workers now
for i in range(8):
planner.add_command(broadcast())
planner.add_command(entrypoint())
for j in range(thread_count):
planner.add_command(Die())
exc = None
def fn(planner: WorkPlanner):
nonlocal exc
try:
while True:
LOG.log(TRACE, "Requesting for a work")
cmd = planner.get_next_command()
LOG.log(TRACE, "The command is received %s [group=%s]",
type(cmd), cmd.group)
if isinstance(cmd, BroadcastHAStates):
time.sleep(1)
if isinstance(cmd, EntrypointRequest):
planner.shutdown()
if isinstance(cmd, Die):
LOG.log(TRACE,
"Poison pill is received - exiting. Bye!")
planner.notify_finished(cmd)
break
tracker.log(cmd)
LOG.log(TRACE, "The job is done, notifying the planner")
planner.notify_finished(cmd)
LOG.log(TRACE, "Notified. ")
except Exception as e:
LOG.exception('*** ERROR ***')
exc = e
workers = [
Thread(target=fn, args=(planner, )) for t in range(thread_count)
]
t0 = time.time()
for t in workers:
t.start()
for t in workers:
t.join()
if exc:
raise exc
tracks = tracker.get_tracks()
idx, (cmd, ts) = self.find(tracks,
lambda a: isinstance(a[0], EntrypointRequest),
'EntrypointRequest not processed')
self.assertTrue(ts - t0 < 3)
self.assertTrue(len(tracks) < 4)
