def test_metrics(self): try: conf = ClusterConf(metrics=0.5) ub_front = 'udp://127.0.0.1:7483' ub_back = 'udp://127.0.0.1:7484' metric_addr = 'udp://127.0.0.1:8845' w_addr = 'udp://127.0.0.1' w_addr_2 = 'udp://127.0.0.1' self.step() a = self._broker(ub_front, ub_back, conf=conf, url_metrics=metric_addr) b = self._worker( w_addr, conf, ub_back, fn=worker_function_sleeper, url_metrics=metric_addr ) c = self.ps.popen(server_main, run_metrics, metric_addr) self.step() with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br: br.assign(RPCKey.new(), Request(5)) self.step() x = {} with self.ps.timer(20) as tr, client_transport(MetricReceiver, metric_addr, ClientConfig(horz=False, timeout_total=3)) as mr: while len(x) <= 1: x = mr.metric_counts() logging.getLogger(__name__).debug(x) tr.sleep(0.3) self.step() with self.ps.timer(20) as tr, client_transport(MetricReceiver, metric_addr, ClientConfig(horz=False, timeout_total=3)) as mr: while len(x) >= 2: x = mr.metric_counts() logging.getLogger(__name__).debug(x) tr.sleep(0.3) self.step() b.send_signal(SIGTERM) c.send_signal(SIGTERM) a.send_signal(SIGTERM) self.assertEqual(wait_all(a, b, c, max_wait=4), [0, 0, 0]) except: logging.getLogger(__name__).exception('Now exiting') raise
def generic_b_main(addr_b, addr_a): rpc = GenericRPC[int] t = Transport.from_url(addr_b, ) with t: ts = EventLoop() tref = ts.transport_add(t) pt = ServiceDefn.from_cls(rpc) r: GenericRPC[int] = build_wrapper(pt, tref, addr_a, conf=ClientConfig(timeout_total=5)) while True: try: a = r.process(5) assert a == Data(5), a break except HorizonPassedError: sleep(0) while True: try: a = r.process_blunt(5) assert a == 5, a return except HorizonPassedError: sleep(0)
def get_metrics(): with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False, timeout_total=3)) as mr: mr: Broker[Request, Response] r = mr.metrics() trc().error('%s', r) return r
def _test_worker_startup(self, par_conf: WorkerConf): conf = ClusterConf() ub = 'udp://127.0.0.1:5678' uw = 'udp://127.0.0.1:5789' pidw = self._worker( uw, conf, ub, worker_function, None, par_conf=par_conf ) with self.ps.timer(20.) as tr, client_transport( Worker[Request, Response], uw, ClientConfig(horz=False)) as br: x: Optional[WorkerMetric] = None slept = 0. # while the WorkerBroker is starting up, it's being bombarded # by loads of messages from Workers. # it's thus missing the message while True: try: x = br.metrics() except TimeoutError: x = None if x is None or x.workers_free < par_conf.processes * par_conf.threads: slept = tr.sleep(0.3) else: trc('slept1').error('%s', slept) break pidw.send_signal(SIGTERM) self.assertEqual(wait_all(pidw, max_wait=2), [0])
def bk_announce(self, wbrid: Optional[RPCKey], load: WorkerLoad): wid = sender() trc('1').debug('%s', wid) if wid not in self.workers: self._worker_new(RPCKey.new(), wid, load) brid = self.workers_brids[wid] if wbrid != brid: s = service(Worker[self.cls_req, self.cls_res], wid, ClientConfig(timeout_total=1)) s.registered(self.workers_brids[wid]) # todo only ping workers through announce wst = self.workers[wid] flag_changed = wst.load.capacity != load.capacity wst.load.capacity = load.capacity self._worker_pings(brid, wid) if flag_changed: self._worker_conf_changed()
def ep(self, a: int) -> int: assert a > 0 s = service(Recursive, origin(), ClientConfig(timeout_total=1., horz=False)) if a == 1: return 0 elif a > 1: return s.ep(a - 1)
def _test(self, cls: Type[Exceptional], mask: List[int]): url_b = f'unix://{self.dtemp}/hugely.sock' url_b_l = f'udp://127.0.0.1:5678' url_a_l = f'udp://127.0.0.1:5679' a = self.ps.popen(server_main_new, lambda: (cls, cls()), { DEFAULT_GROUP: url_b + '#bind', LIVELY: url_b_l, }) with client_transport(Lively, url_b_l, conf=ClientConfig(horz=False, timeout_total=None)) as br: br.is_alive() b = self.ps.popen(server_main_new, lambda: (ExceptionalClient, ExceptionalClient()), { DEFAULT_GROUP: url_b, LIVELY: url_a_l, }) with client_transport(Lively, url_a_l, conf=ClientConfig(horz=False, timeout_total=None)) as br: br.is_alive() b.send_signal(SIGTERM) with client_transport(Lively, url_a_l, conf=ClientConfig(horz=False, timeout_total=1.)) as br: while True: try: br.is_alive() except TimeoutError: break a.send_signal(SIGTERM) self.assertEqual(wait_all(a, b, max_wait=1), mask)
def test_self(self): url_b = 'udp://127.0.0.1:3456' a = self.ps.popen(server_main, recursive_main, url_b) with client_transport(Recursive, url_b, ClientConfig(horz=False)) as b: self.assertEqual(0, b.ep(10)) a.send_signal(SIGTERM) self.assertEqual(wait_all(a, max_wait=1), [0])
def test_callback_service_failure(self): url_b = 'udp://127.0.0.1:32457' b = self.ps.popen(server_main, recursive_b_main, url_b) with client_transport(RecursiveB, url_b, ClientConfig(horz=False)) as acli: with self.assertRaises(ClientTransportCircuitBreaker): acli.count_status() b.send_signal(SIGTERM) self.assertEqual(wait_all(b, max_wait=1), [0])
def main(server_url, conf=ClientConfig(timeout_total=5), **kwargs): service_type = Broker[str, str] T: Type[service_type] = service_type.__class__ t = Transport.from_url('udp://0.0.0.0') with t: ts = EventLoop() ets = ts.transport_add(t) pt = ServiceDefn.from_cls(service_type) r: T = build_wrapper(pt, ets, server_url, conf=conf) print(r.metrics())
def test_double(self): url_a = 'udp://127.0.0.1:32456' url_b = 'udp://127.0.0.1:32457' a = self.ps.popen(server_main, recursive_a_main, url_a, url_b) b = self.ps.popen(server_main, recursive_b_main, url_b) with client_transport(RecursiveA, url_a, ClientConfig(horz=False)) as acli: self.assertEqual(1, acli.poll()) a.send_signal(SIGTERM) b.send_signal(SIGTERM) self.assertEqual(wait_all(a, b, max_wait=1), [0, 0])
def test_reset_str(self): td = mkdtemp() try: ua = f'unix://{td}/sock.sock' a = self.ps.popen(server_main_new, lambda: (ResetStrActor, ResetStrActor()), {DEFAULT_GROUP: ua + '#bind'}) sleep(0.5) with client_transport(ResetStrActor, ua, ClientConfig(horz=False), origin=ua) as acli: self.assertEqual(True, acli.stop()) self.assertEqual(wait_all(a), [0]) finally: shutil.rmtree(td)
def test_double_exc(self): url_a = 'udp://127.0.0.1:42522' url_b = 'udp://127.0.0.1:54352' a = self.ps.popen(server_main, lambda _: (RecursiveC, RecursiveC(url_b)), url_a) b = self.ps.popen(server_main, recursive_b_main, url_b) try: with client_transport(RecursiveA, url_a, ClientConfig(horz=False, timeout_total=2)) as acli: self.assertEqual(1, acli.poll()) except TimeoutError: pass self.assertEqual(wait_all(a, b, max_wait=1), [1, 1])
def test_workflow_b(self): conf = ClusterConf() ub_front = 'udp://127.0.0.1:54546' ub_back = 'udp://127.0.0.1:54540' res_addr = 'udp://127.0.0.1:7485?finished=finished_b' w_addr = 'udp://127.0.0.1' a = self._broker( ub_front, ub_back, conf=conf, url_results=res_addr ) b = self._worker( w_addr, conf=conf, url_broker=ub_back, fn=worker_function ) c = self.ps.popen( server_main_new, lambda: (ResultsReceiver, ResultsReceiver()), { DEFAULT_GROUP: res_addr, } ) with self.ps.timer(5.) as tr, client_transport( Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br: x = 0 while x == 0: x = br.metrics().workers tr.sleep(1.) with client_transport(Broker[Request, Response], ub_front) as br: br.assign(RPCKey.new(), Request(1)) self.ps.wait([c]) b.send_signal(SIGTERM) with self.ps.timer(5.) as tr, client_transport(Broker[Request, Response], ub_front) as br: x = 1 while x > 0: x = br.metrics().workers tr.sleep(1) a.send_signal(SIGTERM) self.assertEqual(wait_all(a, b, c, max_wait=1), [0, 0, 0])
def test_udp(self): rpc = ExemplaryRPC t = Transport.from_url('udp://127.0.0.1:8905') with t: ts = EventLoop() tref = ts.transport_add(t) pt = ServiceDefn.from_cls(rpc) with self.assertRaises(error.TimeoutError): r: ExemplaryRPC = build_wrapper( pt, tref, 'udp://127.0.0.1:7483', conf=ClientConfig(timeout_total=2.)) a = r.move_something(5, 6, 8, pop='asd') b = r.reply(5, 6, 8, pop='asd')
def test_worker_participation(self): conf = ClusterConf() ub_front = 'udp://127.0.0.1:5678' ub_back = 'udp://127.0.0.1:5679' uw1 = 'udp://127.0.0.1' uw2 = 'udp://127.0.0.1' par_conf = WorkerConf(1, 13) pidws = [] for uw in [uw1, uw2]: pidw = self._worker( uw, conf=conf, url_broker=ub_back, fn=worker_function, par_conf=par_conf, ) pidws.append(pidw) pidb = self._broker( ub_front, ub_back, conf=conf, ) with self.ps.timer(5.) as tr, client_transport( Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br: x: BrokerMetric = None while x is None or x.workers < 2: x = br.metrics() tr.sleep(0.05) trc('0').debug(x) for pidw in pidws: pidw.send_signal(SIGTERM) pidb.send_signal(SIGTERM) self.assertEqual(wait_all(*pidws + [pidb]), [0, 0, 0])
def service(obj: Type[T], dest: Origin, conf: Optional[ClientConfig] = None, group=None, **kwargs) -> T: if conf is None: conf = ClientConfig(**kwargs) ctx = context() defn = cache_get(obj) group = ctx.chan_def if group is None else group assert group is not None, 'service() can only be called within execution context that references channels' trc('1').debug('%s %s %s', obj, dest, group) try: chan_idx = ctx.chans[group] except KeyError: raise ValueError(f'{group} not found in {ctx.chans.keys()}') else: return ServiceWrapper(defn, conf, ctx.el.transport(chan_idx), dest)
def exit(self): self.has_terminated = True for wfpid in self.workers_fork_addrs.values(): os.kill(wfpid, SIGTERM) while len(self.workers_free): wid = self.workers_free.pop() wpid = self.worker_addrs[wid] del self.worker_addrs[wid] os.kill(wpid, SIGKILL) self._evict_all(terminating=True) try: s = service(Broker[self.cls_req, self.cls_res], self.url_broker, ClientConfig(timeout_total=1.)) s.bk_unregister() except TimeoutError: logging.getLogger('exit').error('Could not contact broker') raise TerminationException()
def test_resign(self): # todo test double assignment dtemp = self.dtemp conf = ClusterConf(metrics=0.5) ub_front = 'udp://127.0.0.1:7483' ub_back = 'udp://127.0.0.1:7484' metric_addr = 'udp://127.0.0.1:8845' w_addr = 'udp://127.0.0.1:5648' w_addr_2 = 'udp://127.0.0.1' a = self._broker( ub_front, ub_back, conf=conf, url_metrics=metric_addr) b = self._worker( w_addr, conf, ub_back, fn=worker_function_resign, url_metrics=metric_addr ) c = self.ps.popen(server_main, run_metrics, metric_addr) key = RPCKey.new() self.step() ftemp = os.path.join(dtemp, 'toucher') with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br: br: Broker[Request, Response] br.assign(key, Request(5, where=ftemp)) x = {} self.step() with self.ps.timer(20) as tr, client_transport(MetricReceiver, metric_addr, ClientConfig(horz=False, timeout_total=3)) as mr: while len(x) <= 1: x = mr.metric_counts() trc('1').debug(x) tr.sleep(0.3) self.step() with self.ps.timer(20) as tr: tr.sleep(0.35) self.step() with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br: br: Broker[Request, Response] br.resign(key) self.step() with open(ftemp, 'w+') as _: pass with self.ps.timer(20) as tr, client_transport(MetricReceiver, metric_addr, ClientConfig(horz=False, timeout_total=3)) as mr: while len(x) >= 2: x = mr.metric_counts() logging.getLogger(__name__).debug(x) tr.sleep(0.3) with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br: self.assertEqual(0, br.metrics().jobs) b.send_signal(SIGTERM) c.send_signal(SIGTERM) a.send_signal(SIGTERM) self.assertEqual(wait_all(a, b, c, max_wait=5), [0, 0, 0])
def count_status(self, name: str = 'running') -> int: s = service(RecursiveA, sender(), ClientConfig(timeout_total=1., horz=False)) return len([x for x in s.status() if x == name])
def poll(self, name: str = 'running') -> int: s = service(RecursiveB, self.url, ClientConfig(timeout_total=1., horz=False)) return s.count_status(name)
def test_workflow_a(self): conf = ClusterConf() ub_front = 'udp://127.0.0.1:54546' ub_back = 'udp://127.0.0.1:54540' res_addr = 'udp://127.0.0.1:7485?finished=finished_a' w_addr = 'udp://127.0.0.1' brpo = self._broker( ub_front, ub_back, conf=conf, url_results=res_addr ) wrpo = self._worker( w_addr, conf=conf, url_broker=ub_back, fn=worker_function ) repo = self.ps.popen( server_main_new, lambda: (ResultsReceiver, ResultsReceiver()), { DEFAULT_GROUP: res_addr, } ) with self.ps.timer(10.) as tr, client_transport( Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br: x = 0 slept = 0. while True: x = br.metrics().workers if x == 0: slept = tr.sleep(1.) else: trc('slept3').error('%s', slept) break with client_transport(Broker[Request, Response], ub_front) as br: br: Broker[Request, Request] br.assign(RPCKey.new(), Request(1)) self.ps.wait([repo]) wrpo.send_signal(SIGTERM) with self.ps.timer(5.) as tr, client_transport(Broker[Request, Response], ub_front) as br: x = 1 while x > 0: x = br.metrics().workers tr.sleep(1) with self.ps.timer(5.) as tr, client_transport(Broker[Request, Response], ub_front) as br: ms = br.metrics() self.assertEqual(0, ms.jobs) self.assertEqual(0, ms.jobs_pending) self.assertEqual(0, ms.assigned) brpo.send_signal(SIGTERM)
def test_worker_patchup(self): dtemp = self.dtemp conf = ClusterConf( heartbeat=0.5, metrics=0.5, ) par_conf = WorkerConf() ub_front = 'udp://127.0.0.1:7483' ub_back = 'udp://127.0.0.1:7484' w_addr = 'udp://127.0.0.1:5648' w_addr_2 = 'udp://127.0.0.1' a = self._broker( ub_front, ub_back, conf=conf, ) b = self._worker( w_addr, conf, ub_back, fn=worker_function_raise, ) key = RPCKey.new() self.step('assign') ftemp = os.path.join(dtemp, 'toucher') with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br: br: Broker[Request, Response] br.assign(key, Request(5, where=ftemp)) self.step('wait_until_assigned') def get_metrics(): with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False, timeout_total=3)) as mr: mr: Broker[Request, Response] r = mr.metrics() trc().error('%s', r) return r x = get_metrics() with self.ps.timer(20) as tr: while x.jobs < 1: x = get_metrics() tr.sleep(0.01) self.step('job_assigned') with self.ps.timer(20) as tr: tr.sleep(0.35) self.step('create_ftemp') with open(ftemp, 'w+') as _: pass self.step('wait_done') with self.ps.timer(20) as tr: while x.jobs > 0 and x.capacity < par_conf.total: x = get_metrics() tr.sleep(0.01) with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br: self.assertEqual(0, br.metrics().jobs) self.step('wait_term') b.send_signal(SIGTERM) a.send_signal(SIGTERM) self.assertEqual(wait_all(a, b, max_wait=5), [0, 0])
def test_kill_worker(self): conf = ClusterConf(heartbeat=0.5, max_pings=5) ub_front = 'udp://127.0.0.1:54546' ub_back = 'udp://127.0.0.1:54540' res_addr = 'udp://127.0.0.1:54547' w_addr = 'udp://127.0.0.1:54548' a = self._broker(ub_front, ub_back, conf=conf, url_results=res_addr) b = self._worker(w_addr, conf, ub_back, fn=worker_function, url_metrics=None, par_conf=WorkerConf(1, 2)) c = self.ps.popen( server_main_new, lambda: (ResultsReceiver, ResultsReceiver()), { DEFAULT_GROUP: res_addr, } ) logging.getLogger(__name__).warning('A') with self.ps.timer(5.) as tr, client_transport( Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br: x = 0 while x == 0: x = br.metrics().workers trc('1').error('%s', x) tr.sleep(1) logging.getLogger(__name__).warning('B') b.send_signal(SIGKILL) logging.getLogger(__name__).warning('C') with self.ps.timer(5.) as tr, client_transport(Broker[Request, Response], ub_front) as br: br: Broker[Request, Response] x = 1 while x > 0: x = br.metrics().workers trc('2').error('%s', x) tr.sleep(1) self.assertEqual( BrokerMetric( 0, 0, 0, 0, 0, 0, 0, ), br.metrics() ) logging.getLogger(__name__).warning('D') c.send_signal(SIGTERM) a.send_signal(SIGTERM) self.assertEqual(wait_all(a, c, max_wait=1), [0, 0])