def _test_worker_startup(self, par_conf: WorkerConf):
conf = ClusterConf()
ub = 'udp://127.0.0.1:5678'
uw = 'udp://127.0.0.1:5789'
pidw = self._worker(
uw, conf, ub, worker_function, None, par_conf=par_conf
)
with self.ps.timer(20.) as tr, client_transport(
Worker[Request, Response], uw, ClientConfig(horz=False)) as br:
x: Optional[WorkerMetric] = None
slept = 0.
# while the WorkerBroker is starting up, it's being bombarded
# by loads of messages from Workers.
# it's thus missing the message
while True:
try:
x = br.metrics()
except TimeoutError:
x = None
if x is None or x.workers_free < par_conf.processes * par_conf.threads:
slept = tr.sleep(0.3)
else:
trc('slept1').error('%s', slept)
break
pidw.send_signal(SIGTERM)
self.assertEqual(wait_all(pidw, max_wait=2), [0])
def test_metrics(self):
try:
conf = ClusterConf(metrics=0.5)
ub_front = 'udp://127.0.0.1:7483'
ub_back = 'udp://127.0.0.1:7484'
metric_addr = 'udp://127.0.0.1:8845'
w_addr = 'udp://127.0.0.1'
w_addr_2 = 'udp://127.0.0.1'
self.step()
a = self._broker(ub_front, ub_back, conf=conf, url_metrics=metric_addr)
b = self._worker(
w_addr,
conf,
ub_back,
fn=worker_function_sleeper,
url_metrics=metric_addr
)
c = self.ps.popen(server_main, run_metrics, metric_addr)
self.step()
with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
br.assign(RPCKey.new(), Request(5))
self.step()
x = {}
with self.ps.timer(20) as tr, client_transport(MetricReceiver, metric_addr,
ClientConfig(horz=False, timeout_total=3)) as mr:
while len(x) <= 1:
x = mr.metric_counts()
logging.getLogger(__name__).debug(x)
tr.sleep(0.3)
self.step()
with self.ps.timer(20) as tr, client_transport(MetricReceiver, metric_addr,
ClientConfig(horz=False, timeout_total=3)) as mr:
while len(x) >= 2:
x = mr.metric_counts()
logging.getLogger(__name__).debug(x)
tr.sleep(0.3)
self.step()
b.send_signal(SIGTERM)
c.send_signal(SIGTERM)
a.send_signal(SIGTERM)
self.assertEqual(wait_all(a, b, c, max_wait=4), [0, 0, 0])
except:
logging.getLogger(__name__).exception('Now exiting')
raise
def test_workflow_b(self):
conf = ClusterConf()
ub_front = 'udp://127.0.0.1:54546'
ub_back = 'udp://127.0.0.1:54540'
res_addr = 'udp://127.0.0.1:7485?finished=finished_b'
w_addr = 'udp://127.0.0.1'
a = self._broker(
ub_front,
ub_back,
conf=conf,
url_results=res_addr
)
b = self._worker(
w_addr, conf=conf, url_broker=ub_back, fn=worker_function
)
c = self.ps.popen(
server_main_new,
lambda: (ResultsReceiver, ResultsReceiver()),
{
DEFAULT_GROUP: res_addr,
}
)
with self.ps.timer(5.) as tr, client_transport(
Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
x = 0
while x == 0:
x = br.metrics().workers
tr.sleep(1.)
with client_transport(Broker[Request, Response], ub_front) as br:
br.assign(RPCKey.new(), Request(1))
self.ps.wait([c])
b.send_signal(SIGTERM)
with self.ps.timer(5.) as tr, client_transport(Broker[Request, Response], ub_front) as br:
x = 1
while x > 0:
x = br.metrics().workers
tr.sleep(1)
a.send_signal(SIGTERM)
self.assertEqual(wait_all(a, b, c, max_wait=1), [0, 0, 0])
def test_worker_participation(self):
conf = ClusterConf()
ub_front = 'udp://127.0.0.1:5678'
ub_back = 'udp://127.0.0.1:5679'
uw1 = 'udp://127.0.0.1'
uw2 = 'udp://127.0.0.1'
par_conf = WorkerConf(1, 13)
pidws = []
for uw in [uw1, uw2]:
pidw = self._worker(
uw,
conf=conf,
url_broker=ub_back,
fn=worker_function,
par_conf=par_conf,
)
pidws.append(pidw)
pidb = self._broker(
ub_front,
ub_back,
conf=conf,
)
with self.ps.timer(5.) as tr, client_transport(
Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
x: BrokerMetric = None
while x is None or x.workers < 2:
x = br.metrics()
tr.sleep(0.05)
trc('0').debug(x)
for pidw in pidws:
pidw.send_signal(SIGTERM)
pidb.send_signal(SIGTERM)
self.assertEqual(wait_all(*pidws + [pidb]), [0, 0, 0])
def test_backpressure(self):
conf = ClusterConf(heartbeat=0.5, max_pings=5)
backlog = 4
br_conf = BrokerConf(
backlog=backlog,
flush_backlog=0,
)
wr_conf = WorkerConf(1, backlog)
jobs_total = 10
jobs_cancel = 2
ub_front = 'udp://127.0.0.1:54546'
ub_back = 'udp://127.0.0.1:54540'
ur_front = 'udp://127.0.0.1:54547'
uw_front = 'udp://127.0.0.1:54548'
a = self._broker(ub_front, ub_back, conf=conf, url_results=ur_front, par_conf=br_conf)
b = self._worker(uw_front, conf=conf, url_broker=ub_back, fn=worker_function_retry, url_metrics=None,
par_conf=wr_conf)
c = self.ps.popen(
server_main_new,
lambda: (BPResultsReceiver, BPResultsReceiver()),
{
DEFAULT_GROUP: ur_front
}
)
tots = 0
failed = 0
self.step('assign')
flock = os.path.join(self.dtemp, 'lock')
ok_keys = []
with client_transport(Broker[Request, Response], dest=ub_front, horz=False) as br:
br: Broker[Request, Response]
with self.ps.timer(2) as tr:
for i in range(jobs_total):
key = RPCKey.new()
res = br.assign(key, Request(i * 1000, where=flock))
tots += 1
if not res:
failed += 1
else:
ok_keys.append(key)
tr.sleep(0.01)
jobs_to_fail = tots - br_conf.backlog
jobs_to_ok = br_conf.backlog
self.assertEqual(jobs_to_fail, failed)
self.step('check_cancel')
assert jobs_cancel < jobs_to_ok
with client_transport(Broker[Request, Response], dest=ub_front, horz=False) as br:
br: Broker[Request, Response]
with self.ps.timer(2) as tr:
for _, key in zip(range(jobs_cancel), ok_keys):
self.assertEqual(True, br.cancel(key))
jobs_to_ok -= jobs_cancel
self.step('check_backpressure')
with open(flock, 'w+'):
pass
self.step('job_barrier')
x = 0
with client_transport(Broker[Request, Response], dest=ub_front, horz=False) as br:
br: Broker[Request, Response]
with self.ps.timer(2) as tr:
x = 0
while x < jobs_to_ok:
x = br.metrics().flushing
tr.sleep(0.01)
self.assertEqual(jobs_to_ok, x)
self.step('res_check')
hc_now = None
hc_after = None
with client_transport(BPResultsReceiver, dest=ur_front, horz=False) as rr:
rr: BPResultsReceiver
_, hc_now = rr.count()
self.assertLess(0, hc_now)
self.step('res_barrier')
rr.enable()
r = 0
with self.ps.timer(2) as tr:
while r < jobs_to_ok:
r, hc_after = rr.count()
tr.sleep(0.01)
self.assertEqual(jobs_to_ok, r)
self.assertLess(hc_after, hc_now + 5)
b.send_signal(SIGTERM)
c.send_signal(SIGTERM)
a.send_signal(SIGTERM)
self.assertEqual(wait_all(a, b, c, max_wait=5), [0, 0, 0])
def test_worker_patchup(self):
dtemp = self.dtemp
conf = ClusterConf(
heartbeat=0.5,
metrics=0.5,
)
par_conf = WorkerConf()
ub_front = 'udp://127.0.0.1:7483'
ub_back = 'udp://127.0.0.1:7484'
w_addr = 'udp://127.0.0.1:5648'
w_addr_2 = 'udp://127.0.0.1'
a = self._broker(
ub_front,
ub_back,
conf=conf,
)
b = self._worker(
w_addr,
conf,
ub_back,
fn=worker_function_raise,
)
key = RPCKey.new()
self.step('assign')
ftemp = os.path.join(dtemp, 'toucher')
with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
br: Broker[Request, Response]
br.assign(key, Request(5, where=ftemp))
self.step('wait_until_assigned')
def get_metrics():
with client_transport(Broker[Request, Response], ub_front,
ClientConfig(horz=False, timeout_total=3)) as mr:
mr: Broker[Request, Response]
r = mr.metrics()
trc().error('%s', r)
return r
x = get_metrics()
with self.ps.timer(20) as tr:
while x.jobs < 1:
x = get_metrics()
tr.sleep(0.01)
self.step('job_assigned')
with self.ps.timer(20) as tr:
tr.sleep(0.35)
self.step('create_ftemp')
with open(ftemp, 'w+') as _:
pass
self.step('wait_done')
with self.ps.timer(20) as tr:
while x.jobs > 0 and x.capacity < par_conf.total:
x = get_metrics()
tr.sleep(0.01)
with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
self.assertEqual(0, br.metrics().jobs)
self.step('wait_term')
b.send_signal(SIGTERM)
a.send_signal(SIGTERM)
self.assertEqual(wait_all(a, b, max_wait=5), [0, 0])
def test_resign(self):
# todo test double assignment
dtemp = self.dtemp
conf = ClusterConf(metrics=0.5)
ub_front = 'udp://127.0.0.1:7483'
ub_back = 'udp://127.0.0.1:7484'
metric_addr = 'udp://127.0.0.1:8845'
w_addr = 'udp://127.0.0.1:5648'
w_addr_2 = 'udp://127.0.0.1'
a = self._broker(
ub_front,
ub_back,
conf=conf,
url_metrics=metric_addr)
b = self._worker(
w_addr,
conf,
ub_back,
fn=worker_function_resign,
url_metrics=metric_addr
)
c = self.ps.popen(server_main, run_metrics, metric_addr)
key = RPCKey.new()
self.step()
ftemp = os.path.join(dtemp, 'toucher')
with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
br: Broker[Request, Response]
br.assign(key, Request(5, where=ftemp))
x = {}
self.step()
with self.ps.timer(20) as tr, client_transport(MetricReceiver, metric_addr,
ClientConfig(horz=False, timeout_total=3)) as mr:
while len(x) <= 1:
x = mr.metric_counts()
trc('1').debug(x)
tr.sleep(0.3)
self.step()
with self.ps.timer(20) as tr:
tr.sleep(0.35)
self.step()
with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
br: Broker[Request, Response]
br.resign(key)
self.step()
with open(ftemp, 'w+') as _:
pass
with self.ps.timer(20) as tr, client_transport(MetricReceiver, metric_addr,
ClientConfig(horz=False, timeout_total=3)) as mr:
while len(x) >= 2:
x = mr.metric_counts()
logging.getLogger(__name__).debug(x)
tr.sleep(0.3)
with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
self.assertEqual(0, br.metrics().jobs)
b.send_signal(SIGTERM)
c.send_signal(SIGTERM)
a.send_signal(SIGTERM)
self.assertEqual(wait_all(a, b, c, max_wait=5), [0, 0, 0])
def test_kill_worker(self):
conf = ClusterConf(heartbeat=0.5, max_pings=5)
ub_front = 'udp://127.0.0.1:54546'
ub_back = 'udp://127.0.0.1:54540'
res_addr = 'udp://127.0.0.1:54547'
w_addr = 'udp://127.0.0.1:54548'
a = self._broker(ub_front, ub_back, conf=conf, url_results=res_addr)
b = self._worker(w_addr, conf, ub_back, fn=worker_function, url_metrics=None, par_conf=WorkerConf(1, 2))
c = self.ps.popen(
server_main_new,
lambda: (ResultsReceiver, ResultsReceiver()),
{
DEFAULT_GROUP: res_addr,
}
)
logging.getLogger(__name__).warning('A')
with self.ps.timer(5.) as tr, client_transport(
Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
x = 0
while x == 0:
x = br.metrics().workers
trc('1').error('%s', x)
tr.sleep(1)
logging.getLogger(__name__).warning('B')
b.send_signal(SIGKILL)
logging.getLogger(__name__).warning('C')
with self.ps.timer(5.) as tr, client_transport(Broker[Request, Response], ub_front) as br:
br: Broker[Request, Response]
x = 1
while x > 0:
x = br.metrics().workers
trc('2').error('%s', x)
tr.sleep(1)
self.assertEqual(
BrokerMetric(
0,
0,
0,
0,
0,
0,
0,
),
br.metrics()
)
logging.getLogger(__name__).warning('D')
c.send_signal(SIGTERM)
a.send_signal(SIGTERM)
self.assertEqual(wait_all(a, c, max_wait=1), [0, 0])
def test_workflow_a(self):
conf = ClusterConf()
ub_front = 'udp://127.0.0.1:54546'
ub_back = 'udp://127.0.0.1:54540'
res_addr = 'udp://127.0.0.1:7485?finished=finished_a'
w_addr = 'udp://127.0.0.1'
brpo = self._broker(
ub_front,
ub_back,
conf=conf,
url_results=res_addr
)
wrpo = self._worker(
w_addr,
conf=conf,
url_broker=ub_back,
fn=worker_function
)
repo = self.ps.popen(
server_main_new,
lambda: (ResultsReceiver, ResultsReceiver()),
{
DEFAULT_GROUP: res_addr,
}
)
with self.ps.timer(10.) as tr, client_transport(
Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
x = 0
slept = 0.
while True:
x = br.metrics().workers
if x == 0:
slept = tr.sleep(1.)
else:
trc('slept3').error('%s', slept)
break
with client_transport(Broker[Request, Response], ub_front) as br:
br: Broker[Request, Request]
br.assign(RPCKey.new(), Request(1))
self.ps.wait([repo])
wrpo.send_signal(SIGTERM)
with self.ps.timer(5.) as tr, client_transport(Broker[Request, Response], ub_front) as br:
x = 1
while x > 0:
x = br.metrics().workers
tr.sleep(1)
with self.ps.timer(5.) as tr, client_transport(Broker[Request, Response], ub_front) as br:
ms = br.metrics()
self.assertEqual(0, ms.jobs)
self.assertEqual(0, ms.jobs_pending)
self.assertEqual(0, ms.assigned)
brpo.send_signal(SIGTERM)