def put(self, payload: Optional[RequestType]): s: WorkerThreadInst[self.cls_req, self.cls_res] = service( WorkerThreadInst[self.cls_req, self.cls_res], self.thread_addr, group=BACKEND, ) s.put(payload)
def push_announce(self) -> float: s = service(Broker[self.cls_req, self.cls_res], self.url_broker, group=DEFAULT_GROUP) s.bk_announce(self.brid, self.load) return self.conf.heartbeat
def bk_announce(self, is_fork: bool, wpid: int): sdr = sender() trc('0').error('%s %s %s', sdr, is_fork, wpid) if is_fork: if sdr in self.workers_fork_addrs: return else: self.workers_fork_addrs[sdr] = wpid s = service(WorkerForkInst, sdr, group=BACKEND) for _ in range(self.par_conf.threads): trc('01').error('%s', _) s.fork() return if sdr in self.worker_addrs: return trc('1').debug('%s', sdr) self.worker_addrs[sdr] = wpid self.workers_free.push(sdr) self.load.capacity += 1 if self.load.capacity == self.par_conf.processes * self.par_conf.threads: trc('capa').debug('capacity reached') self._started() if self.has_started: self.push_announce()
def bk_announce(self, wbrid: Optional[RPCKey], load: WorkerLoad): wid = sender() trc('1').debug('%s', wid) if wid not in self.workers: self._worker_new(RPCKey.new(), wid, load) brid = self.workers_brids[wid] if wbrid != brid: s = service(Worker[self.cls_req, self.cls_res], wid, ClientConfig(timeout_total=1)) s.registered(self.workers_brids[wid]) # todo only ping workers through announce wst = self.workers[wid] flag_changed = wst.load.capacity != load.capacity wst.load.capacity = load.capacity self._worker_pings(brid, wid) if flag_changed: self._worker_conf_changed()
def bk_done(self, brid: RPCKey, jid: RPCKey, ok: bool, res: Optional[ResponseType] = None): w = service(Worker[self.cls_req, self.cls_res], sender()) w.done_ack(brid, jid) if self._brid_check(brid): return if jid not in self.jobs: trc('1').error('%s', jid) return if self.jobs[jid].finished: trc('2').warning('%s', jid) return if jid in self.jobs_pending_assign: del self.jobs_pending_assign[jid] if jid in self.jobs_pending_resign: del self.jobs_pending_resign[jid] if jid not in self.jobs_workers or sender() != self.jobs_workers[jid]: return self._worker_pings(brid, sender()) if ok: self._job_done(jid, res) else: self._job_resign(jid)
def _push_resign(self, jid: RPCKey) -> Optional[float]: wid = self.jobs_workers[jid] brid = self.workers_brids[wid] s = service(Worker[self.cls_req, self.cls_res], wid) s.resign(brid, jid) return self.conf.retry_delta
def _push_done(self, jid: RPCKey): sb = service(Broker[self.cls_req, self.cls_res], self.url_broker) if jid in self.jobs: if jid in self.jobs_res: sb.bk_done(self.brid, jid, True, self.jobs_res[jid]) else: sb.bk_done(self.brid, jid, False) return self.conf.retry_delta
def ep(self, a: int) -> int: assert a > 0 s = service(Recursive, origin(), ClientConfig(timeout_total=1., horz=False)) if a == 1: return 0 elif a > 1: return s.ep(a - 1)
def push_fork(self) -> Optional[float]: while len(self.workers_fork): wid = self.workers_fork.popleft() if self.load.capacity >= self.par_conf.total: continue s = service(WorkerForkInst, wid, group=BACKEND) # this specific call _may_ cause the event loop to re-loop. s.fork() return None
def _push_flush(self, jid: RPCKey) -> Optional[float]: if not self.url_results: self.flush_ack(jid) else: assert self.jobs[jid].finished s = service(BrokerResult[self.cls_res], self.url_results, group=DEFAULT_GROUP) s: BrokerResult[ResponseType] s.finished(jid, self.jobs[jid].res) return self.conf.retry_delta
def assign(self, brid: RPCKey, jid: RPCKey, jreq: RequestType): # todo: currently sends to an address will cause an exception if we're lucky enough to # todo: send a task right after the worker had died (and before transport realised that) sb = service(Broker[self.cls_req, self.cls_res], self.url_broker) if brid != self.brid: trc('brid').error('%s != %s %s', brid, self.brid, jid) reset(self.push_announce, 0) sb.bk_assign(brid, jid, False) return if jid in self.jobs: # is there any scenario where a job may be assigned to something else ? trc('kno').error('%s', jid) sb.bk_assign(brid, jid, True) return if len(self.workers_free) == 0 or len( self.jobs_res) >= self.conf.pending_max: sb.bk_assign(brid, jid, False) return nwid = self.workers_free.pop() self.jobs[jid] = jreq self.jobs_workers[jid] = nwid self.workers_jobs[nwid] = jid s = service(WorkerInst[self.cls_req, self.cls_res], nwid, group=BACKEND) s.put(jreq) self.load.occupied += 1 sb.bk_assign(brid, jid, True)
def broadcast(self) -> float: for x in self.origins: c = service(BroadcastClientRPC, x) c.ping() for k in list(self.origins.keys()): self.origins[k] -= 1 if self.origins[k] <= 0: del self.origins[k] logging.getLogger(__name__ + '.' + self.__class__.__name__).info( f'%d %s %s', len(self.origins_met), self.origins_met, self.origins) if len(self.origins_met) == 1 and len(self.origins) == 0: raise TerminationException() return 0.05
def exit(self): self.has_terminated = True for wfpid in self.workers_fork_addrs.values(): os.kill(wfpid, SIGTERM) while len(self.workers_free): wid = self.workers_free.pop() wpid = self.worker_addrs[wid] del self.worker_addrs[wid] os.kill(wpid, SIGKILL) self._evict_all(terminating=True) try: s = service(Broker[self.cls_req, self.cls_res], self.url_broker, ClientConfig(timeout_total=1.)) s.bk_unregister() except TimeoutError: logging.getLogger('exit').error('Could not contact broker') raise TerminationException()
def announce(self) -> Optional[float]: s = service(WorkerAnnounce, origin()) s.bk_announce(False, os.getpid()) return None
def poll(self, name: str = 'running') -> int: s = service(RecursiveB, self.url, ClientConfig(timeout_total=1., horz=False)) return s.count_status(name)
def count_status(self, name: str = 'running') -> int: s = service(RecursiveA, sender(), ClientConfig(timeout_total=1., horz=False)) return len([x for x in s.status() if x == name])
def _finished_ack(jid: RPCKey): s = service(BrokerFlushAck, sender(), group=DEFAULT_GROUP) s.flush_ack(jid)
def push_metrics(self) -> float: s = service(MetricCollector, self.url_metrics) s.metrics(self.metrics()) return self.conf.metrics
def push_metrics(self) -> float: # todo worker should not expose any of the jobs running to the metrics, this is a broker's job if self.url_metrics: s = service(MetricCollector, self.url_metrics) s.metrics(self.metrics()) return self.conf.metrics if self.url_metrics else None
def init(self) -> Optional[float]: x: WorkerInst[self.cls_req, self.cls_res] = service( WorkerInst[self.cls_req, self.cls_res], origin()) x.bk_started() return None
def put(self, payload: RequestType): ret = self.fn(payload) x: WorkerInst[self.cls_req, self.cls_res] = service( WorkerInst[self.cls_req, self.cls_res], origin()) x.bk_done(ret)
def bk_done(self, ret: ResponseType): s = service(Worker[self.cls_req, self.cls_res], origin(DEFAULT_GROUP), group=DEFAULT_GROUP) s.bk_done(ret)
def broadcast(self) -> float: while True: s = service(BroadcastRPC, self.broadcast_addr) s.arrived() return 0.05
def reg(self) -> float: s = service(A1, url_a) s.mopo() return 2.
def _ack(self, jid: RPCKey): s = service(Broker[Request, Response], sender(), group=DEFAULT_GROUP) s: Broker[Request, Response] s.flush_ack(jid)