def put(self, payload: Optional[RequestType]):
s: WorkerThreadInst[self.cls_req, self.cls_res] = service(
WorkerThreadInst[self.cls_req, self.cls_res],
self.thread_addr,
group=BACKEND,
)
s.put(payload)
def push_announce(self) -> float:
s = service(Broker[self.cls_req, self.cls_res],
self.url_broker,
group=DEFAULT_GROUP)
s.bk_announce(self.brid, self.load)
return self.conf.heartbeat
def bk_announce(self, is_fork: bool, wpid: int):
sdr = sender()
trc('0').error('%s %s %s', sdr, is_fork, wpid)
if is_fork:
if sdr in self.workers_fork_addrs:
return
else:
self.workers_fork_addrs[sdr] = wpid
s = service(WorkerForkInst, sdr, group=BACKEND)
for _ in range(self.par_conf.threads):
trc('01').error('%s', _)
s.fork()
return
if sdr in self.worker_addrs:
return
trc('1').debug('%s', sdr)
self.worker_addrs[sdr] = wpid
self.workers_free.push(sdr)
self.load.capacity += 1
if self.load.capacity == self.par_conf.processes * self.par_conf.threads:
trc('capa').debug('capacity reached')
self._started()
if self.has_started:
self.push_announce()
def bk_announce(self, wbrid: Optional[RPCKey], load: WorkerLoad):
wid = sender()
trc('1').debug('%s', wid)
if wid not in self.workers:
self._worker_new(RPCKey.new(), wid, load)
brid = self.workers_brids[wid]
if wbrid != brid:
s = service(Worker[self.cls_req, self.cls_res], wid,
ClientConfig(timeout_total=1))
s.registered(self.workers_brids[wid])
# todo only ping workers through announce
wst = self.workers[wid]
flag_changed = wst.load.capacity != load.capacity
wst.load.capacity = load.capacity
self._worker_pings(brid, wid)
if flag_changed:
self._worker_conf_changed()
def bk_done(self,
brid: RPCKey,
jid: RPCKey,
ok: bool,
res: Optional[ResponseType] = None):
w = service(Worker[self.cls_req, self.cls_res], sender())
w.done_ack(brid, jid)
if self._brid_check(brid):
return
if jid not in self.jobs:
trc('1').error('%s', jid)
return
if self.jobs[jid].finished:
trc('2').warning('%s', jid)
return
if jid in self.jobs_pending_assign:
del self.jobs_pending_assign[jid]
if jid in self.jobs_pending_resign:
del self.jobs_pending_resign[jid]
if jid not in self.jobs_workers or sender() != self.jobs_workers[jid]:
return
self._worker_pings(brid, sender())
if ok:
self._job_done(jid, res)
else:
self._job_resign(jid)
def _push_resign(self, jid: RPCKey) -> Optional[float]:
wid = self.jobs_workers[jid]
brid = self.workers_brids[wid]
s = service(Worker[self.cls_req, self.cls_res], wid)
s.resign(brid, jid)
return self.conf.retry_delta
def _push_done(self, jid: RPCKey):
sb = service(Broker[self.cls_req, self.cls_res], self.url_broker)
if jid in self.jobs:
if jid in self.jobs_res:
sb.bk_done(self.brid, jid, True, self.jobs_res[jid])
else:
sb.bk_done(self.brid, jid, False)
return self.conf.retry_delta
def ep(self, a: int) -> int:
assert a > 0
s = service(Recursive, origin(),
ClientConfig(timeout_total=1., horz=False))
if a == 1:
return 0
elif a > 1:
return s.ep(a - 1)
def push_fork(self) -> Optional[float]:
while len(self.workers_fork):
wid = self.workers_fork.popleft()
if self.load.capacity >= self.par_conf.total:
continue
s = service(WorkerForkInst, wid, group=BACKEND)
# this specific call _may_ cause the event loop to re-loop.
s.fork()
return None
def _push_flush(self, jid: RPCKey) -> Optional[float]:
if not self.url_results:
self.flush_ack(jid)
else:
assert self.jobs[jid].finished
s = service(BrokerResult[self.cls_res],
self.url_results,
group=DEFAULT_GROUP)
s: BrokerResult[ResponseType]
s.finished(jid, self.jobs[jid].res)
return self.conf.retry_delta
def assign(self, brid: RPCKey, jid: RPCKey, jreq: RequestType):
# todo: currently sends to an address will cause an exception if we're lucky enough to
# todo: send a task right after the worker had died (and before transport realised that)
sb = service(Broker[self.cls_req, self.cls_res], self.url_broker)
if brid != self.brid:
trc('brid').error('%s != %s %s', brid, self.brid, jid)
reset(self.push_announce, 0)
sb.bk_assign(brid, jid, False)
return
if jid in self.jobs:
# is there any scenario where a job may be assigned to something else ?
trc('kno').error('%s', jid)
sb.bk_assign(brid, jid, True)
return
if len(self.workers_free) == 0 or len(
self.jobs_res) >= self.conf.pending_max:
sb.bk_assign(brid, jid, False)
return
nwid = self.workers_free.pop()
self.jobs[jid] = jreq
self.jobs_workers[jid] = nwid
self.workers_jobs[nwid] = jid
s = service(WorkerInst[self.cls_req, self.cls_res],
nwid,
group=BACKEND)
s.put(jreq)
self.load.occupied += 1
sb.bk_assign(brid, jid, True)
def broadcast(self) -> float:
for x in self.origins:
c = service(BroadcastClientRPC, x)
c.ping()
for k in list(self.origins.keys()):
self.origins[k] -= 1
if self.origins[k] <= 0:
del self.origins[k]
logging.getLogger(__name__ + '.' + self.__class__.__name__).info(
f'%d %s %s',
len(self.origins_met), self.origins_met, self.origins)
if len(self.origins_met) == 1 and len(self.origins) == 0:
raise TerminationException()
return 0.05
def exit(self):
self.has_terminated = True
for wfpid in self.workers_fork_addrs.values():
os.kill(wfpid, SIGTERM)
while len(self.workers_free):
wid = self.workers_free.pop()
wpid = self.worker_addrs[wid]
del self.worker_addrs[wid]
os.kill(wpid, SIGKILL)
self._evict_all(terminating=True)
try:
s = service(Broker[self.cls_req, self.cls_res], self.url_broker,
ClientConfig(timeout_total=1.))
s.bk_unregister()
except TimeoutError:
logging.getLogger('exit').error('Could not contact broker')
raise TerminationException()
def announce(self) -> Optional[float]:
s = service(WorkerAnnounce, origin())
s.bk_announce(False, os.getpid())
return None
def poll(self, name: str = 'running') -> int:
s = service(RecursiveB, self.url,
ClientConfig(timeout_total=1., horz=False))
return s.count_status(name)
def count_status(self, name: str = 'running') -> int:
s = service(RecursiveA, sender(),
ClientConfig(timeout_total=1., horz=False))
return len([x for x in s.status() if x == name])
def _finished_ack(jid: RPCKey):
s = service(BrokerFlushAck, sender(), group=DEFAULT_GROUP)
s.flush_ack(jid)
def push_metrics(self) -> float:
s = service(MetricCollector, self.url_metrics)
s.metrics(self.metrics())
return self.conf.metrics
def push_metrics(self) -> float:
# todo worker should not expose any of the jobs running to the metrics, this is a broker's job
if self.url_metrics:
s = service(MetricCollector, self.url_metrics)
s.metrics(self.metrics())
return self.conf.metrics if self.url_metrics else None
def init(self) -> Optional[float]:
x: WorkerInst[self.cls_req, self.cls_res] = service(
WorkerInst[self.cls_req, self.cls_res], origin())
x.bk_started()
return None
def put(self, payload: RequestType):
ret = self.fn(payload)
x: WorkerInst[self.cls_req, self.cls_res] = service(
WorkerInst[self.cls_req, self.cls_res], origin())
x.bk_done(ret)
def bk_done(self, ret: ResponseType):
s = service(Worker[self.cls_req, self.cls_res],
origin(DEFAULT_GROUP),
group=DEFAULT_GROUP)
s.bk_done(ret)
def broadcast(self) -> float:
while True:
s = service(BroadcastRPC, self.broadcast_addr)
s.arrived()
return 0.05
def reg(self) -> float:
s = service(A1, url_a)
s.mopo()
return 2.
def _ack(self, jid: RPCKey):
s = service(Broker[Request, Response], sender(), group=DEFAULT_GROUP)
s: Broker[Request, Response]
s.flush_ack(jid)
