def test_metrics(self):
try:
conf = ClusterConf(metrics=0.5)
ub_front = 'udp://127.0.0.1:7483'
ub_back = 'udp://127.0.0.1:7484'
metric_addr = 'udp://127.0.0.1:8845'
w_addr = 'udp://127.0.0.1'
w_addr_2 = 'udp://127.0.0.1'
self.step()
a = self._broker(ub_front, ub_back, conf=conf, url_metrics=metric_addr)
b = self._worker(
w_addr,
conf,
ub_back,
fn=worker_function_sleeper,
url_metrics=metric_addr
)
c = self.ps.popen(server_main, run_metrics, metric_addr)
self.step()
with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
br.assign(RPCKey.new(), Request(5))
self.step()
x = {}
with self.ps.timer(20) as tr, client_transport(MetricReceiver, metric_addr,
ClientConfig(horz=False, timeout_total=3)) as mr:
while len(x) <= 1:
x = mr.metric_counts()
logging.getLogger(__name__).debug(x)
tr.sleep(0.3)
self.step()
with self.ps.timer(20) as tr, client_transport(MetricReceiver, metric_addr,
ClientConfig(horz=False, timeout_total=3)) as mr:
while len(x) >= 2:
x = mr.metric_counts()
logging.getLogger(__name__).debug(x)
tr.sleep(0.3)
self.step()
b.send_signal(SIGTERM)
c.send_signal(SIGTERM)
a.send_signal(SIGTERM)
self.assertEqual(wait_all(a, b, c, max_wait=4), [0, 0, 0])
except:
logging.getLogger(__name__).exception('Now exiting')
raise
def generic_b_main(addr_b, addr_a):
rpc = GenericRPC[int]
t = Transport.from_url(addr_b, )
with t:
ts = EventLoop()
tref = ts.transport_add(t)
pt = ServiceDefn.from_cls(rpc)
r: GenericRPC[int] = build_wrapper(pt,
tref,
addr_a,
conf=ClientConfig(timeout_total=5))
while True:
try:
a = r.process(5)
assert a == Data(5), a
break
except HorizonPassedError:
sleep(0)
while True:
try:
a = r.process_blunt(5)
assert a == 5, a
return
except HorizonPassedError:
sleep(0)
def get_metrics():
with client_transport(Broker[Request, Response], ub_front,
ClientConfig(horz=False, timeout_total=3)) as mr:
mr: Broker[Request, Response]
r = mr.metrics()
trc().error('%s', r)
return r
def _test_worker_startup(self, par_conf: WorkerConf):
conf = ClusterConf()
ub = 'udp://127.0.0.1:5678'
uw = 'udp://127.0.0.1:5789'
pidw = self._worker(
uw, conf, ub, worker_function, None, par_conf=par_conf
)
with self.ps.timer(20.) as tr, client_transport(
Worker[Request, Response], uw, ClientConfig(horz=False)) as br:
x: Optional[WorkerMetric] = None
slept = 0.
# while the WorkerBroker is starting up, it's being bombarded
# by loads of messages from Workers.
# it's thus missing the message
while True:
try:
x = br.metrics()
except TimeoutError:
x = None
if x is None or x.workers_free < par_conf.processes * par_conf.threads:
slept = tr.sleep(0.3)
else:
trc('slept1').error('%s', slept)
break
pidw.send_signal(SIGTERM)
self.assertEqual(wait_all(pidw, max_wait=2), [0])
def bk_announce(self, wbrid: Optional[RPCKey], load: WorkerLoad):
wid = sender()
trc('1').debug('%s', wid)
if wid not in self.workers:
self._worker_new(RPCKey.new(), wid, load)
brid = self.workers_brids[wid]
if wbrid != brid:
s = service(Worker[self.cls_req, self.cls_res], wid,
ClientConfig(timeout_total=1))
s.registered(self.workers_brids[wid])
# todo only ping workers through announce
wst = self.workers[wid]
flag_changed = wst.load.capacity != load.capacity
wst.load.capacity = load.capacity
self._worker_pings(brid, wid)
if flag_changed:
self._worker_conf_changed()
def ep(self, a: int) -> int:
assert a > 0
s = service(Recursive, origin(),
ClientConfig(timeout_total=1., horz=False))
if a == 1:
return 0
elif a > 1:
return s.ep(a - 1)
def _test(self, cls: Type[Exceptional], mask: List[int]):
url_b = f'unix://{self.dtemp}/hugely.sock'
url_b_l = f'udp://127.0.0.1:5678'
url_a_l = f'udp://127.0.0.1:5679'
a = self.ps.popen(server_main_new, lambda: (cls, cls()), {
DEFAULT_GROUP: url_b + '#bind',
LIVELY: url_b_l,
})
with client_transport(Lively,
url_b_l,
conf=ClientConfig(horz=False,
timeout_total=None)) as br:
br.is_alive()
b = self.ps.popen(server_main_new, lambda:
(ExceptionalClient, ExceptionalClient()), {
DEFAULT_GROUP: url_b,
LIVELY: url_a_l,
})
with client_transport(Lively,
url_a_l,
conf=ClientConfig(horz=False,
timeout_total=None)) as br:
br.is_alive()
b.send_signal(SIGTERM)
with client_transport(Lively,
url_a_l,
conf=ClientConfig(horz=False,
timeout_total=1.)) as br:
while True:
try:
br.is_alive()
except TimeoutError:
break
a.send_signal(SIGTERM)
self.assertEqual(wait_all(a, b, max_wait=1), mask)
def test_self(self):
url_b = 'udp://127.0.0.1:3456'
a = self.ps.popen(server_main, recursive_main, url_b)
with client_transport(Recursive, url_b, ClientConfig(horz=False)) as b:
self.assertEqual(0, b.ep(10))
a.send_signal(SIGTERM)
self.assertEqual(wait_all(a, max_wait=1), [0])
def test_callback_service_failure(self):
url_b = 'udp://127.0.0.1:32457'
b = self.ps.popen(server_main, recursive_b_main, url_b)
with client_transport(RecursiveB, url_b,
ClientConfig(horz=False)) as acli:
with self.assertRaises(ClientTransportCircuitBreaker):
acli.count_status()
b.send_signal(SIGTERM)
self.assertEqual(wait_all(b, max_wait=1), [0])
def main(server_url, conf=ClientConfig(timeout_total=5), **kwargs):
service_type = Broker[str, str]
T: Type[service_type] = service_type.__class__
t = Transport.from_url('udp://0.0.0.0')
with t:
ts = EventLoop()
ets = ts.transport_add(t)
pt = ServiceDefn.from_cls(service_type)
r: T = build_wrapper(pt, ets, server_url, conf=conf)
print(r.metrics())
def test_double(self):
url_a = 'udp://127.0.0.1:32456'
url_b = 'udp://127.0.0.1:32457'
a = self.ps.popen(server_main, recursive_a_main, url_a, url_b)
b = self.ps.popen(server_main, recursive_b_main, url_b)
with client_transport(RecursiveA, url_a,
ClientConfig(horz=False)) as acli:
self.assertEqual(1, acli.poll())
a.send_signal(SIGTERM)
b.send_signal(SIGTERM)
self.assertEqual(wait_all(a, b, max_wait=1), [0, 0])
def test_reset_str(self):
td = mkdtemp()
try:
ua = f'unix://{td}/sock.sock'
a = self.ps.popen(server_main_new, lambda: (ResetStrActor, ResetStrActor()), {DEFAULT_GROUP: ua + '#bind'})
sleep(0.5)
with client_transport(ResetStrActor, ua, ClientConfig(horz=False), origin=ua) as acli:
self.assertEqual(True, acli.stop())
self.assertEqual(wait_all(a), [0])
finally:
shutil.rmtree(td)
def test_double_exc(self):
url_a = 'udp://127.0.0.1:42522'
url_b = 'udp://127.0.0.1:54352'
a = self.ps.popen(server_main, lambda _:
(RecursiveC, RecursiveC(url_b)), url_a)
b = self.ps.popen(server_main, recursive_b_main, url_b)
try:
with client_transport(RecursiveA, url_a,
ClientConfig(horz=False,
timeout_total=2)) as acli:
self.assertEqual(1, acli.poll())
except TimeoutError:
pass
self.assertEqual(wait_all(a, b, max_wait=1), [1, 1])
def test_workflow_b(self):
conf = ClusterConf()
ub_front = 'udp://127.0.0.1:54546'
ub_back = 'udp://127.0.0.1:54540'
res_addr = 'udp://127.0.0.1:7485?finished=finished_b'
w_addr = 'udp://127.0.0.1'
a = self._broker(
ub_front,
ub_back,
conf=conf,
url_results=res_addr
)
b = self._worker(
w_addr, conf=conf, url_broker=ub_back, fn=worker_function
)
c = self.ps.popen(
server_main_new,
lambda: (ResultsReceiver, ResultsReceiver()),
{
DEFAULT_GROUP: res_addr,
}
)
with self.ps.timer(5.) as tr, client_transport(
Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
x = 0
while x == 0:
x = br.metrics().workers
tr.sleep(1.)
with client_transport(Broker[Request, Response], ub_front) as br:
br.assign(RPCKey.new(), Request(1))
self.ps.wait([c])
b.send_signal(SIGTERM)
with self.ps.timer(5.) as tr, client_transport(Broker[Request, Response], ub_front) as br:
x = 1
while x > 0:
x = br.metrics().workers
tr.sleep(1)
a.send_signal(SIGTERM)
self.assertEqual(wait_all(a, b, c, max_wait=1), [0, 0, 0])
def test_udp(self):
rpc = ExemplaryRPC
t = Transport.from_url('udp://127.0.0.1:8905')
with t:
ts = EventLoop()
tref = ts.transport_add(t)
pt = ServiceDefn.from_cls(rpc)
with self.assertRaises(error.TimeoutError):
r: ExemplaryRPC = build_wrapper(
pt,
tref,
'udp://127.0.0.1:7483',
conf=ClientConfig(timeout_total=2.))
a = r.move_something(5, 6, 8, pop='asd')
b = r.reply(5, 6, 8, pop='asd')
def test_worker_participation(self):
conf = ClusterConf()
ub_front = 'udp://127.0.0.1:5678'
ub_back = 'udp://127.0.0.1:5679'
uw1 = 'udp://127.0.0.1'
uw2 = 'udp://127.0.0.1'
par_conf = WorkerConf(1, 13)
pidws = []
for uw in [uw1, uw2]:
pidw = self._worker(
uw,
conf=conf,
url_broker=ub_back,
fn=worker_function,
par_conf=par_conf,
)
pidws.append(pidw)
pidb = self._broker(
ub_front,
ub_back,
conf=conf,
)
with self.ps.timer(5.) as tr, client_transport(
Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
x: BrokerMetric = None
while x is None or x.workers < 2:
x = br.metrics()
tr.sleep(0.05)
trc('0').debug(x)
for pidw in pidws:
pidw.send_signal(SIGTERM)
pidb.send_signal(SIGTERM)
self.assertEqual(wait_all(*pidws + [pidb]), [0, 0, 0])
def service(obj: Type[T], dest: Origin, conf: Optional[ClientConfig] = None, group=None, **kwargs) -> T:
if conf is None:
conf = ClientConfig(**kwargs)
ctx = context()
defn = cache_get(obj)
group = ctx.chan_def if group is None else group
assert group is not None, 'service() can only be called within execution context that references channels'
trc('1').debug('%s %s %s', obj, dest, group)
try:
chan_idx = ctx.chans[group]
except KeyError:
raise ValueError(f'{group} not found in {ctx.chans.keys()}')
else:
return ServiceWrapper(defn, conf, ctx.el.transport(chan_idx), dest)
def exit(self):
self.has_terminated = True
for wfpid in self.workers_fork_addrs.values():
os.kill(wfpid, SIGTERM)
while len(self.workers_free):
wid = self.workers_free.pop()
wpid = self.worker_addrs[wid]
del self.worker_addrs[wid]
os.kill(wpid, SIGKILL)
self._evict_all(terminating=True)
try:
s = service(Broker[self.cls_req, self.cls_res], self.url_broker,
ClientConfig(timeout_total=1.))
s.bk_unregister()
except TimeoutError:
logging.getLogger('exit').error('Could not contact broker')
raise TerminationException()
def test_resign(self):
# todo test double assignment
dtemp = self.dtemp
conf = ClusterConf(metrics=0.5)
ub_front = 'udp://127.0.0.1:7483'
ub_back = 'udp://127.0.0.1:7484'
metric_addr = 'udp://127.0.0.1:8845'
w_addr = 'udp://127.0.0.1:5648'
w_addr_2 = 'udp://127.0.0.1'
a = self._broker(
ub_front,
ub_back,
conf=conf,
url_metrics=metric_addr)
b = self._worker(
w_addr,
conf,
ub_back,
fn=worker_function_resign,
url_metrics=metric_addr
)
c = self.ps.popen(server_main, run_metrics, metric_addr)
key = RPCKey.new()
self.step()
ftemp = os.path.join(dtemp, 'toucher')
with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
br: Broker[Request, Response]
br.assign(key, Request(5, where=ftemp))
x = {}
self.step()
with self.ps.timer(20) as tr, client_transport(MetricReceiver, metric_addr,
ClientConfig(horz=False, timeout_total=3)) as mr:
while len(x) <= 1:
x = mr.metric_counts()
trc('1').debug(x)
tr.sleep(0.3)
self.step()
with self.ps.timer(20) as tr:
tr.sleep(0.35)
self.step()
with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
br: Broker[Request, Response]
br.resign(key)
self.step()
with open(ftemp, 'w+') as _:
pass
with self.ps.timer(20) as tr, client_transport(MetricReceiver, metric_addr,
ClientConfig(horz=False, timeout_total=3)) as mr:
while len(x) >= 2:
x = mr.metric_counts()
logging.getLogger(__name__).debug(x)
tr.sleep(0.3)
with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
self.assertEqual(0, br.metrics().jobs)
b.send_signal(SIGTERM)
c.send_signal(SIGTERM)
a.send_signal(SIGTERM)
self.assertEqual(wait_all(a, b, c, max_wait=5), [0, 0, 0])
def count_status(self, name: str = 'running') -> int:
s = service(RecursiveA, sender(),
ClientConfig(timeout_total=1., horz=False))
return len([x for x in s.status() if x == name])
def poll(self, name: str = 'running') -> int:
s = service(RecursiveB, self.url,
ClientConfig(timeout_total=1., horz=False))
return s.count_status(name)
def test_workflow_a(self):
conf = ClusterConf()
ub_front = 'udp://127.0.0.1:54546'
ub_back = 'udp://127.0.0.1:54540'
res_addr = 'udp://127.0.0.1:7485?finished=finished_a'
w_addr = 'udp://127.0.0.1'
brpo = self._broker(
ub_front,
ub_back,
conf=conf,
url_results=res_addr
)
wrpo = self._worker(
w_addr,
conf=conf,
url_broker=ub_back,
fn=worker_function
)
repo = self.ps.popen(
server_main_new,
lambda: (ResultsReceiver, ResultsReceiver()),
{
DEFAULT_GROUP: res_addr,
}
)
with self.ps.timer(10.) as tr, client_transport(
Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
x = 0
slept = 0.
while True:
x = br.metrics().workers
if x == 0:
slept = tr.sleep(1.)
else:
trc('slept3').error('%s', slept)
break
with client_transport(Broker[Request, Response], ub_front) as br:
br: Broker[Request, Request]
br.assign(RPCKey.new(), Request(1))
self.ps.wait([repo])
wrpo.send_signal(SIGTERM)
with self.ps.timer(5.) as tr, client_transport(Broker[Request, Response], ub_front) as br:
x = 1
while x > 0:
x = br.metrics().workers
tr.sleep(1)
with self.ps.timer(5.) as tr, client_transport(Broker[Request, Response], ub_front) as br:
ms = br.metrics()
self.assertEqual(0, ms.jobs)
self.assertEqual(0, ms.jobs_pending)
self.assertEqual(0, ms.assigned)
brpo.send_signal(SIGTERM)
def test_worker_patchup(self):
dtemp = self.dtemp
conf = ClusterConf(
heartbeat=0.5,
metrics=0.5,
)
par_conf = WorkerConf()
ub_front = 'udp://127.0.0.1:7483'
ub_back = 'udp://127.0.0.1:7484'
w_addr = 'udp://127.0.0.1:5648'
w_addr_2 = 'udp://127.0.0.1'
a = self._broker(
ub_front,
ub_back,
conf=conf,
)
b = self._worker(
w_addr,
conf,
ub_back,
fn=worker_function_raise,
)
key = RPCKey.new()
self.step('assign')
ftemp = os.path.join(dtemp, 'toucher')
with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
br: Broker[Request, Response]
br.assign(key, Request(5, where=ftemp))
self.step('wait_until_assigned')
def get_metrics():
with client_transport(Broker[Request, Response], ub_front,
ClientConfig(horz=False, timeout_total=3)) as mr:
mr: Broker[Request, Response]
r = mr.metrics()
trc().error('%s', r)
return r
x = get_metrics()
with self.ps.timer(20) as tr:
while x.jobs < 1:
x = get_metrics()
tr.sleep(0.01)
self.step('job_assigned')
with self.ps.timer(20) as tr:
tr.sleep(0.35)
self.step('create_ftemp')
with open(ftemp, 'w+') as _:
pass
self.step('wait_done')
with self.ps.timer(20) as tr:
while x.jobs > 0 and x.capacity < par_conf.total:
x = get_metrics()
tr.sleep(0.01)
with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
self.assertEqual(0, br.metrics().jobs)
self.step('wait_term')
b.send_signal(SIGTERM)
a.send_signal(SIGTERM)
self.assertEqual(wait_all(a, b, max_wait=5), [0, 0])
def test_kill_worker(self):
conf = ClusterConf(heartbeat=0.5, max_pings=5)
ub_front = 'udp://127.0.0.1:54546'
ub_back = 'udp://127.0.0.1:54540'
res_addr = 'udp://127.0.0.1:54547'
w_addr = 'udp://127.0.0.1:54548'
a = self._broker(ub_front, ub_back, conf=conf, url_results=res_addr)
b = self._worker(w_addr, conf, ub_back, fn=worker_function, url_metrics=None, par_conf=WorkerConf(1, 2))
c = self.ps.popen(
server_main_new,
lambda: (ResultsReceiver, ResultsReceiver()),
{
DEFAULT_GROUP: res_addr,
}
)
logging.getLogger(__name__).warning('A')
with self.ps.timer(5.) as tr, client_transport(
Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
x = 0
while x == 0:
x = br.metrics().workers
trc('1').error('%s', x)
tr.sleep(1)
logging.getLogger(__name__).warning('B')
b.send_signal(SIGKILL)
logging.getLogger(__name__).warning('C')
with self.ps.timer(5.) as tr, client_transport(Broker[Request, Response], ub_front) as br:
br: Broker[Request, Response]
x = 1
while x > 0:
x = br.metrics().workers
trc('2').error('%s', x)
tr.sleep(1)
self.assertEqual(
BrokerMetric(
0,
0,
0,
0,
0,
0,
0,
),
br.metrics()
)
logging.getLogger(__name__).warning('D')
c.send_signal(SIGTERM)
a.send_signal(SIGTERM)
self.assertEqual(wait_all(a, c, max_wait=1), [0, 0])
