def bk_announce(self, wbrid: Optional[RPCKey], load: WorkerLoad):
wid = sender()
trc('1').debug('%s', wid)
if wid not in self.workers:
self._worker_new(RPCKey.new(), wid, load)
brid = self.workers_brids[wid]
if wbrid != brid:
s = service(Worker[self.cls_req, self.cls_res], wid,
ClientConfig(timeout_total=1))
s.registered(self.workers_brids[wid])
# todo only ping workers through announce
wst = self.workers[wid]
flag_changed = wst.load.capacity != load.capacity
wst.load.capacity = load.capacity
self._worker_pings(brid, wid)
if flag_changed:
self._worker_conf_changed()
def test_packet(self):
k = RPCKey.new()
a = RPCPacket(k, RPCPacketType.Req, 'абра', [{}, None])
x = a.pack()
b = RPCPacket.unpack(x)
self.assertEqual(a, b)
def bk_register(self, load: WorkerLoad) -> Optional[RPCKey]:
wid = sender()
if wid in self.workers:
# a worker needs to be unregistered first
return None
wbrid = RPCKey.new()
self._worker_new(wbrid, wid, load)
def test_metrics(self):
try:
conf = ClusterConf(metrics=0.5)
ub_front = 'udp://127.0.0.1:7483'
ub_back = 'udp://127.0.0.1:7484'
metric_addr = 'udp://127.0.0.1:8845'
w_addr = 'udp://127.0.0.1'
w_addr_2 = 'udp://127.0.0.1'
self.step()
a = self._broker(ub_front, ub_back, conf=conf, url_metrics=metric_addr)
b = self._worker(
w_addr,
conf,
ub_back,
fn=worker_function_sleeper,
url_metrics=metric_addr
)
c = self.ps.popen(server_main, run_metrics, metric_addr)
self.step()
with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
br.assign(RPCKey.new(), Request(5))
self.step()
x = {}
with self.ps.timer(20) as tr, client_transport(MetricReceiver, metric_addr,
ClientConfig(horz=False, timeout_total=3)) as mr:
while len(x) <= 1:
x = mr.metric_counts()
logging.getLogger(__name__).debug(x)
tr.sleep(0.3)
self.step()
with self.ps.timer(20) as tr, client_transport(MetricReceiver, metric_addr,
ClientConfig(horz=False, timeout_total=3)) as mr:
while len(x) >= 2:
x = mr.metric_counts()
logging.getLogger(__name__).debug(x)
tr.sleep(0.3)
self.step()
b.send_signal(SIGTERM)
c.send_signal(SIGTERM)
a.send_signal(SIGTERM)
self.assertEqual(wait_all(a, b, c, max_wait=4), [0, 0, 0])
except:
logging.getLogger(__name__).exception('Now exiting')
raise
def test_rpclogdict(self):
cr = time_now()
x = RPCLogDict(cr)
with self.subTest('a'):
with self.assertRaises(HorizonPassedError):
x[RPCKey(time_now() - timedelta(seconds=10))] = False
kv = RPCKey()
with self.subTest('b'):
val = True
x[kv] = val
self.assertEqual(x[kv], val)
with self.subTest('c'):
x.set_horizon(time_now())
with self.assertRaises(HorizonPassedError):
x[kv]
def test_workflow_b(self):
conf = ClusterConf()
ub_front = 'udp://127.0.0.1:54546'
ub_back = 'udp://127.0.0.1:54540'
res_addr = 'udp://127.0.0.1:7485?finished=finished_b'
w_addr = 'udp://127.0.0.1'
a = self._broker(
ub_front,
ub_back,
conf=conf,
url_results=res_addr
)
b = self._worker(
w_addr, conf=conf, url_broker=ub_back, fn=worker_function
)
c = self.ps.popen(
server_main_new,
lambda: (ResultsReceiver, ResultsReceiver()),
{
DEFAULT_GROUP: res_addr,
}
)
with self.ps.timer(5.) as tr, client_transport(
Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
x = 0
while x == 0:
x = br.metrics().workers
tr.sleep(1.)
with client_transport(Broker[Request, Response], ub_front) as br:
br.assign(RPCKey.new(), Request(1))
self.ps.wait([c])
b.send_signal(SIGTERM)
with self.ps.timer(5.) as tr, client_transport(Broker[Request, Response], ub_front) as br:
x = 1
while x > 0:
x = br.metrics().workers
tr.sleep(1)
a.send_signal(SIGTERM)
self.assertEqual(wait_all(a, b, c, max_wait=1), [0, 0, 0])
def test_key(self):
a = RPCKey.new()
ap = a.pack()
b = RPCKey.unpack(ap)
self.assertEqual(a, b)
def test_backpressure(self):
conf = ClusterConf(heartbeat=0.5, max_pings=5)
backlog = 4
br_conf = BrokerConf(
backlog=backlog,
flush_backlog=0,
)
wr_conf = WorkerConf(1, backlog)
jobs_total = 10
jobs_cancel = 2
ub_front = 'udp://127.0.0.1:54546'
ub_back = 'udp://127.0.0.1:54540'
ur_front = 'udp://127.0.0.1:54547'
uw_front = 'udp://127.0.0.1:54548'
a = self._broker(ub_front, ub_back, conf=conf, url_results=ur_front, par_conf=br_conf)
b = self._worker(uw_front, conf=conf, url_broker=ub_back, fn=worker_function_retry, url_metrics=None,
par_conf=wr_conf)
c = self.ps.popen(
server_main_new,
lambda: (BPResultsReceiver, BPResultsReceiver()),
{
DEFAULT_GROUP: ur_front
}
)
tots = 0
failed = 0
self.step('assign')
flock = os.path.join(self.dtemp, 'lock')
ok_keys = []
with client_transport(Broker[Request, Response], dest=ub_front, horz=False) as br:
br: Broker[Request, Response]
with self.ps.timer(2) as tr:
for i in range(jobs_total):
key = RPCKey.new()
res = br.assign(key, Request(i * 1000, where=flock))
tots += 1
if not res:
failed += 1
else:
ok_keys.append(key)
tr.sleep(0.01)
jobs_to_fail = tots - br_conf.backlog
jobs_to_ok = br_conf.backlog
self.assertEqual(jobs_to_fail, failed)
self.step('check_cancel')
assert jobs_cancel < jobs_to_ok
with client_transport(Broker[Request, Response], dest=ub_front, horz=False) as br:
br: Broker[Request, Response]
with self.ps.timer(2) as tr:
for _, key in zip(range(jobs_cancel), ok_keys):
self.assertEqual(True, br.cancel(key))
jobs_to_ok -= jobs_cancel
self.step('check_backpressure')
with open(flock, 'w+'):
pass
self.step('job_barrier')
x = 0
with client_transport(Broker[Request, Response], dest=ub_front, horz=False) as br:
br: Broker[Request, Response]
with self.ps.timer(2) as tr:
x = 0
while x < jobs_to_ok:
x = br.metrics().flushing
tr.sleep(0.01)
self.assertEqual(jobs_to_ok, x)
self.step('res_check')
hc_now = None
hc_after = None
with client_transport(BPResultsReceiver, dest=ur_front, horz=False) as rr:
rr: BPResultsReceiver
_, hc_now = rr.count()
self.assertLess(0, hc_now)
self.step('res_barrier')
rr.enable()
r = 0
with self.ps.timer(2) as tr:
while r < jobs_to_ok:
r, hc_after = rr.count()
tr.sleep(0.01)
self.assertEqual(jobs_to_ok, r)
self.assertLess(hc_after, hc_now + 5)
b.send_signal(SIGTERM)
c.send_signal(SIGTERM)
a.send_signal(SIGTERM)
self.assertEqual(wait_all(a, b, c, max_wait=5), [0, 0, 0])
def test_worker_patchup(self):
dtemp = self.dtemp
conf = ClusterConf(
heartbeat=0.5,
metrics=0.5,
)
par_conf = WorkerConf()
ub_front = 'udp://127.0.0.1:7483'
ub_back = 'udp://127.0.0.1:7484'
w_addr = 'udp://127.0.0.1:5648'
w_addr_2 = 'udp://127.0.0.1'
a = self._broker(
ub_front,
ub_back,
conf=conf,
)
b = self._worker(
w_addr,
conf,
ub_back,
fn=worker_function_raise,
)
key = RPCKey.new()
self.step('assign')
ftemp = os.path.join(dtemp, 'toucher')
with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
br: Broker[Request, Response]
br.assign(key, Request(5, where=ftemp))
self.step('wait_until_assigned')
def get_metrics():
with client_transport(Broker[Request, Response], ub_front,
ClientConfig(horz=False, timeout_total=3)) as mr:
mr: Broker[Request, Response]
r = mr.metrics()
trc().error('%s', r)
return r
x = get_metrics()
with self.ps.timer(20) as tr:
while x.jobs < 1:
x = get_metrics()
tr.sleep(0.01)
self.step('job_assigned')
with self.ps.timer(20) as tr:
tr.sleep(0.35)
self.step('create_ftemp')
with open(ftemp, 'w+') as _:
pass
self.step('wait_done')
with self.ps.timer(20) as tr:
while x.jobs > 0 and x.capacity < par_conf.total:
x = get_metrics()
tr.sleep(0.01)
with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
self.assertEqual(0, br.metrics().jobs)
self.step('wait_term')
b.send_signal(SIGTERM)
a.send_signal(SIGTERM)
self.assertEqual(wait_all(a, b, max_wait=5), [0, 0])
def test_resign(self):
# todo test double assignment
dtemp = self.dtemp
conf = ClusterConf(metrics=0.5)
ub_front = 'udp://127.0.0.1:7483'
ub_back = 'udp://127.0.0.1:7484'
metric_addr = 'udp://127.0.0.1:8845'
w_addr = 'udp://127.0.0.1:5648'
w_addr_2 = 'udp://127.0.0.1'
a = self._broker(
ub_front,
ub_back,
conf=conf,
url_metrics=metric_addr)
b = self._worker(
w_addr,
conf,
ub_back,
fn=worker_function_resign,
url_metrics=metric_addr
)
c = self.ps.popen(server_main, run_metrics, metric_addr)
key = RPCKey.new()
self.step()
ftemp = os.path.join(dtemp, 'toucher')
with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
br: Broker[Request, Response]
br.assign(key, Request(5, where=ftemp))
x = {}
self.step()
with self.ps.timer(20) as tr, client_transport(MetricReceiver, metric_addr,
ClientConfig(horz=False, timeout_total=3)) as mr:
while len(x) <= 1:
x = mr.metric_counts()
trc('1').debug(x)
tr.sleep(0.3)
self.step()
with self.ps.timer(20) as tr:
tr.sleep(0.35)
self.step()
with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
br: Broker[Request, Response]
br.resign(key)
self.step()
with open(ftemp, 'w+') as _:
pass
with self.ps.timer(20) as tr, client_transport(MetricReceiver, metric_addr,
ClientConfig(horz=False, timeout_total=3)) as mr:
while len(x) >= 2:
x = mr.metric_counts()
logging.getLogger(__name__).debug(x)
tr.sleep(0.3)
with client_transport(Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
self.assertEqual(0, br.metrics().jobs)
b.send_signal(SIGTERM)
c.send_signal(SIGTERM)
a.send_signal(SIGTERM)
self.assertEqual(wait_all(a, b, c, max_wait=5), [0, 0, 0])
def test_workflow_a(self):
conf = ClusterConf()
ub_front = 'udp://127.0.0.1:54546'
ub_back = 'udp://127.0.0.1:54540'
res_addr = 'udp://127.0.0.1:7485?finished=finished_a'
w_addr = 'udp://127.0.0.1'
brpo = self._broker(
ub_front,
ub_back,
conf=conf,
url_results=res_addr
)
wrpo = self._worker(
w_addr,
conf=conf,
url_broker=ub_back,
fn=worker_function
)
repo = self.ps.popen(
server_main_new,
lambda: (ResultsReceiver, ResultsReceiver()),
{
DEFAULT_GROUP: res_addr,
}
)
with self.ps.timer(10.) as tr, client_transport(
Broker[Request, Response], ub_front, ClientConfig(horz=False)) as br:
x = 0
slept = 0.
while True:
x = br.metrics().workers
if x == 0:
slept = tr.sleep(1.)
else:
trc('slept3').error('%s', slept)
break
with client_transport(Broker[Request, Response], ub_front) as br:
br: Broker[Request, Request]
br.assign(RPCKey.new(), Request(1))
self.ps.wait([repo])
wrpo.send_signal(SIGTERM)
with self.ps.timer(5.) as tr, client_transport(Broker[Request, Response], ub_front) as br:
x = 1
while x > 0:
x = br.metrics().workers
tr.sleep(1)
with self.ps.timer(5.) as tr, client_transport(Broker[Request, Response], ub_front) as br:
ms = br.metrics()
self.assertEqual(0, ms.jobs)
self.assertEqual(0, ms.jobs_pending)
self.assertEqual(0, ms.assigned)
brpo.send_signal(SIGTERM)