def test_no_hang_when_group_id_cycled(self): planner = WorkPlanner() def my_state(): return State(next_group_id=99999, active_commands=LinkedList(), current_group_id=99999, next_group_commands=set(), is_shutdown=False) planner = WorkPlanner(init_state_factory=my_state) tracker = GroupTracker() thread_count = 1 for i in range(10): planner.add_command(process_event()) for j in range(thread_count): planner.add_command(Die()) exc = None def fn(planner: WorkPlanner): nonlocal exc try: while True: LOG.log(TRACE, "Requesting for a work") # import pudb.remote # pudb.remote.set_trace(term_size=(120, 40), port=9998) cmd = planner.get_next_command() LOG.log(TRACE, "The command is received %s [group=%s]", type(cmd), cmd.group) if isinstance(cmd, Die): LOG.log(TRACE, "Poison pill is received - exiting. Bye!") planner.notify_finished(cmd) break tracker.log(cmd) LOG.log(TRACE, "The job is done, notifying the planner") planner.notify_finished(cmd) LOG.log(TRACE, "Notified. ") except Exception as e: LOG.exception('*** ERROR ***') exc = e workers = [ Thread(target=fn, args=(planner, )) for t in range(thread_count) ] for t in workers: t.start() for t in workers: t.join() if exc: raise exc groups_processed = tracker.get_tracks() self.assertEqual([99999, 10**5, 0, 1, 2, 3, 4, 5, 6, 7], groups_processed)
def test_ha_nvec_get_shares_group_always(self): planner = WorkPlanner() assign = planner._assign_group msgs_after_bc = [ assign(broadcast()), assign(nvec_get()), assign(broadcast()), assign(entrypoint()) ] msgs_after_ep = [ assign(entrypoint()), assign(nvec_get()), assign(broadcast()), assign(entrypoint()) ] msgs_after_nvec = [ assign(entrypoint()), assign(nvec_get()), assign(nvec_get()), assign(entrypoint()) ] self.assertEqual([0, 0, 1, 2], [m.group for m in msgs_after_bc]) self.assertEqual([2, 2, 3, 4], [m.group for m in msgs_after_ep]) self.assertEqual([4, 4, 4, 4], [m.group for m in msgs_after_nvec])
def main(): # Note: no logging must happen before this call. # Otherwise the log configuration will not apply. setup_logging() # [KN] The elements in the queue will appear if # 1. A callback is invoked from ha_link (this will happen in a motr # thread which must be free ASAP) # 2. A new HA notification has come form Consul via HTTP # [KN] The messages are consumed by Python thread created by # _run_qconsumer_thread function. # # [KN] Note: The server is launched in the main thread. planner = WorkPlanner() util: ConsulUtil = ConsulUtil() _remove_stale_session(util) cfg: HL_Fids = _get_motr_fids(util) LOG.info('Welcome to HaX') LOG.info(f'Setting up ha_link interface with the options as follows: ' f'hax fid = {cfg.hax_fid}, hax endpoint = {cfg.hax_ep}, ' f'HA fid = {cfg.ha_fid}') ffi = HaxFFI() herald = DeliveryHerald() motr = Motr(planner=planner, ffi=ffi, herald=herald, consul_util=util) # Note that consumer thread must be started before we invoke motr.start(..) # Reason: hax process will send entrypoint request and somebody needs # to reply it. # TODO make the number of threads configurable consumer_threads = [ _run_qconsumer_thread(planner, motr, herald, util, i) for i in range(4) ] try: # [KN] We use just the first profile for Spiel API for now. motr.start(cfg.hax_ep, process=cfg.hax_fid, ha_service=cfg.ha_fid, profile=cfg.profiles[0]) LOG.info('Motr API has been started') rconfc_starter = _run_rconfc_starter_thread(motr, consul_util=util) stats_updater = _run_stats_updater_thread(motr, consul_util=util) event_poller = _run_thread(create_ha_thread(planner, util)) # [KN] This is a blocking call. It will work until the program is # terminated by signal server = ServerRunner(planner, herald, consul_util=util) server.run(threads_to_wait=[ *consumer_threads, stats_updater, rconfc_starter, event_poller ]) except Exception: LOG.exception('Exiting due to an exception') finally: motr.fini()
def test_entrypoint_not_paralleled_with_broadcast(self): planner = WorkPlanner() bcast = broadcast() ep1 = entrypoint() bcast = planner._assign_group(bcast) ep1 = planner._assign_group(ep1) self.assertEqual([0, 1], [bcast.group, ep1.group])
def test_entrypoint_requests_share_same_group(self): planner = WorkPlanner() ep1 = entrypoint() ep2 = entrypoint() ep1 = planner._assign_group(ep1) ep2 = planner._assign_group(ep2) self.assertEqual([0, 0], [ep1.group, ep2.group])
def test_entrypoint_executed_asap(self): planner = WorkPlanner() a = planner._assign_group self.assertEqual( [0, 1, 0], [a(i)[0].group for i in [broadcast(), broadcast(), entrypoint()]])
def test_broadcast_starts_new_group(self): planner = WorkPlanner() assign = planner._assign_group msgs = [ assign(broadcast()), assign(broadcast()), assign(broadcast()), assign(entrypoint()) ] self.assertEqual([0, 1, 2, 3], [m.group for m in msgs])
def test_broadcast_does_not_start_new_group(self): planner = WorkPlanner() assign = planner._assign_group msgs = [ assign(broadcast()), assign(broadcast()), assign(broadcast()), assign(nvec_get()) ] self.assertEqual([0, 1, 2, 0], [m.group for (m, _) in msgs])
def test_parallelism_is_possible(self): planner = WorkPlanner() for i in range(40): planner.add_command(entrypoint()) for j in range(4): planner.add_command(Die()) exc = None def fn(planner: WorkPlanner): nonlocal exc try: while True: LOG.log(TRACE, "Requesting for a work") cmd = planner.get_next_command() LOG.log(TRACE, "The command is received") if isinstance(cmd, Die): LOG.log(TRACE, "Poison pill is received - exiting. Bye!") break sleep(0.5) LOG.log(TRACE, "The job is done, notifying the planner") planner.notify_finished(cmd) LOG.log(TRACE, "Notified. ") except Exception as e: LOG.exception('*** ERROR ***') exc = e workers = [Thread(target=fn, args=(planner, )) for t in range(4)] time_1 = time.time() for t in workers: t.start() for t in workers: t.join() time_2 = time.time() logging.info('Processing time %s', time_2 - time_1) if exc: raise exc self.assertTrue(planner.is_empty(), 'Not all commands were read out') # Every thread sleeps for 500ms. 40 commands * 0.5 gives 20 seconds if # the commands executed sequentially self.assertLess(time_2 - time_1, 19, 'Suspiciously slow')
def test_group_id_cycled(self): def my_state(): return State(next_group_id=99999, active_commands=LinkedList(), current_group_id=99999, next_group_commands=set(), is_shutdown=False) planner = WorkPlanner(init_state_factory=my_state) assign = planner._assign_group msgs = [ assign(broadcast()), assign(broadcast()), assign(broadcast()), assign(broadcast()), ] self.assertEqual([99999, 10**5, 0, 1], [m.group for (m, _) in msgs])
def main(): # Note: no logging must happen before this call. # Otherwise the log configuration will not apply. setup_logging() set_locale() inject.configure(di_configuration) state = inject.instance(HaxGlobalState) # [KN] The elements in the work planner will appear if # 1. A callback is invoked from ha_link (this will happen in a motr # thread which must be free ASAP) # 2. A new HA notification has come form Consul via HTTP # [KN] The messages are consumed by Python threads created by # _run_qconsumer_thread function. # # [KN] Note: The server is launched in the main thread. planner = WorkPlanner() def handle_signal(sig, frame): state.set_stopping() planner.shutdown() # This is necessary to allow hax to exit early if Consul is not available # (otherwise _get_motr_fids() may be retrying forever even if the hax # process needs to shutdown). signal.signal(signal.SIGINT, handle_signal) util: ConsulUtil = ConsulUtil() # Avoid removing session on hax start as this will happen # on every node, thus leader election will keep re-triggering # until the final hax node starts, this will delay further # bootstrapping operations. _remove_stale_session(util) cfg: HL_Fids = _get_motr_fids(util) hax_http_port = util.get_hax_http_port() util.init_motr_processes_status() LOG.info('Welcome to HaX') LOG.info(f'Setting up ha_link interface with the options as follows: ' f'hax fid = {cfg.hax_fid}, hax endpoint = {cfg.hax_ep}, ' f'HA fid = {cfg.ha_fid}') ffi = HaxFFI() herald = DeliveryHerald() motr = Motr(planner=planner, ffi=ffi, herald=herald, consul_util=util) # Note that consumer thread must be started before we invoke motr.start(..) # Reason: hax process will send entrypoint request and somebody needs # to reply it. # TODO make the number of threads configurable consumer_threads = [ _run_qconsumer_thread(planner, motr, herald, util, i) for i in range(32) ] try: # [KN] We use just the first profile for Spiel API for now. motr.start(cfg.hax_ep, process=cfg.hax_fid, ha_service=cfg.ha_fid, profile=cfg.profiles[0]) LOG.info('Motr API has been started') rconfc_starter = _run_rconfc_starter_thread(motr, consul_util=util) stats_updater = _run_stats_updater_thread(motr, consul_util=util) bc_updater = _run_bc_updater_thread(motr, consul_util=util) event_poller = _run_thread(create_ha_thread(planner, util)) # [KN] This is a blocking call. It will work until the program is # terminated by signal server = ServerRunner(planner, herald, consul_util=util, hax_state=state) server.run(threads_to_wait=[ *consumer_threads, stats_updater, bc_updater, rconfc_starter, event_poller ], port=hax_http_port) except Exception: LOG.exception('Exiting due to an exception') finally: motr.fini()
def test_groups_processed_sequentially_4_threads(self): planner = WorkPlanner() group_idx = 0 def ret_values(cmd: BaseMessage) -> bool: nonlocal group_idx # We don't care about the group distribution logic # in this test. Instead, we concentrate how different group # numbers are processed by the workers and the order # in which they are allowed to process the messages. # # _assign_group is invoked under a lock acquired, so this # increment is thread-safe. values = [0, 1, 0, 1, 0, 1, 0, 1, 0, 1] ret = bool(values[group_idx]) group_idx += 1 return ret setattr(planner, '_should_increase_group', Mock(side_effect=ret_values)) tracker = GroupTracker() thread_count = 4 for i in range(10): planner.add_command(entrypoint()) for j in range(thread_count): planner.add_command(Die()) exc = None def fn(planner: WorkPlanner): nonlocal exc try: while True: LOG.log(TRACE, "Requesting for a work") cmd = planner.get_next_command() LOG.log(TRACE, "The command is received %s [group=%s]", type(cmd), cmd.group) if isinstance(cmd, Die): LOG.log(TRACE, "Poison pill is received - exiting. Bye!") planner.notify_finished(cmd) break tracker.log(cmd) LOG.log(TRACE, "The job is done, notifying the planner") planner.notify_finished(cmd) LOG.log(TRACE, "Notified. ") except Exception as e: LOG.exception('*** ERROR ***') exc = e workers = [ Thread(target=fn, args=(planner, )) for t in range(thread_count) ] for t in workers: t.start() for t in workers: t.join() if exc: raise exc groups_processed = tracker.get_tracks() self.assertEqual([0, 1, 1, 2, 2, 3, 3, 4, 4, 5], groups_processed)
def test_workers_not_blocked_by_future_work(self): planner = WorkPlanner() group_idx = 0 tracker = TimeTracker() thread_count = 2 # We add way more commands than we have workers now for i in range(8): planner.add_command(broadcast()) planner.add_command(entrypoint()) for j in range(thread_count): planner.add_command(Die()) exc = None def fn(planner: WorkPlanner): nonlocal exc try: while True: LOG.log(TRACE, "Requesting for a work") cmd = planner.get_next_command() LOG.log(TRACE, "The command is received %s [group=%s]", type(cmd), cmd.group) if isinstance(cmd, BroadcastHAStates): time.sleep(1) if isinstance(cmd, EntrypointRequest): planner.shutdown() if isinstance(cmd, Die): LOG.log(TRACE, "Poison pill is received - exiting. Bye!") planner.notify_finished(cmd) break tracker.log(cmd) LOG.log(TRACE, "The job is done, notifying the planner") planner.notify_finished(cmd) LOG.log(TRACE, "Notified. ") except Exception as e: LOG.exception('*** ERROR ***') exc = e workers = [ Thread(target=fn, args=(planner, )) for t in range(thread_count) ] t0 = time.time() for t in workers: t.start() for t in workers: t.join() if exc: raise exc tracks = tracker.get_tracks() idx, (cmd, ts) = self.find(tracks, lambda a: isinstance(a[0], EntrypointRequest), 'EntrypointRequest not processed') self.assertTrue(ts - t0 < 3) self.assertTrue(len(tracks) < 4)
def test_entrypoint_request_processed_asap(self): planner = WorkPlanner() group_idx = 0 def ret_values(cmd: BaseMessage) -> bool: nonlocal group_idx # We don't care about the group distribution logic # in this test. Instead, we concentrate how different group # numbers are processed by the workers and the order # in which they are allowed to process the messages. # # _assign_group is invoked under a lock acquired, so this # increment is thread-safe. values = [0, 1, 0, 1, 0, 1, 0, 1, 0, 1] ret = bool(values[group_idx]) group_idx += 1 return ret tracker = TimeTracker() thread_count = 4 planner.add_command(broadcast()) planner.add_command(broadcast()) planner.add_command(broadcast()) planner.add_command(entrypoint()) for j in range(thread_count): planner.add_command(Die()) exc = None def fn(planner: WorkPlanner): nonlocal exc try: while True: LOG.log(TRACE, "Requesting for a work") cmd = planner.get_next_command() LOG.log(TRACE, "The command is received %s [group=%s]", type(cmd), cmd.group) if isinstance(cmd, BroadcastHAStates): time.sleep(1.5) if isinstance(cmd, Die): LOG.log(TRACE, "Poison pill is received - exiting. Bye!") planner.notify_finished(cmd) break tracker.log(cmd) LOG.log(TRACE, "The job is done, notifying the planner") planner.notify_finished(cmd) LOG.log(TRACE, "Notified. ") except Exception as e: LOG.exception('*** ERROR ***') exc = e workers = [ Thread(target=fn, args=(planner, )) for t in range(thread_count) ] for t in workers: t.start() for t in workers: t.join() if exc: raise exc tracks = tracker.get_tracks() (cmd, ts) = tracks[0] self.assertTrue(isinstance(cmd, EntrypointRequest))