def step(self, polled_flags): if any(polled_flags) and polled_flags == self.flgs: self.flgs_ctr += 1 elif self.flgs is None: self.flgs = polled_flags self.flgs_ctr = 0 self.time = time_now() else: self.flgs = polled_flags self.flgs_ctr = 0 self.time = time_now() fa = any(self.flgs) and self.flgs_ctr > self.max_ctr vb = fa and ((time_now() - self.time).total_seconds() / self.flgs_ctr) fb = fa and vb < self.max_iter_secs if fa and fb: lgr = self.par.logger('spinlock') lgr.warning('%s %s %s', self.flgs_ctr, self.flgs, vb) lgr.warning('%s', self.par.nonflat_fds()) lgr.warning('%s', self.par.transports) lgr.warning('%s', self.par.waits) _log_traceback(lgr, 'We are in a spinlock') time.sleep(self.sleep_secs)
def loop(self, max_wait: Optional[float] = None): ts = time_now() def bin_none(a, b, fn=min): if a is None: return b elif b is None: return a else: return fn(a, b) elsld = ELSpinLockDetector(self) while True: running_for = (time_now() - ts).total_seconds() max_wait_left = None if max_wait is None else max( max_wait - running_for, 0) max_wait_owned: Optional[float] = None max_waits_curr = self.max_waits for k, x in max_waits_curr.items(): max_wait_owned = bin_none(max_wait_owned, x) if x is not None and x <= 0.: wait = self.waits.get(k) if wait is None: continue with exc_handler(wait.exc): wait.processor() max_wait_step = bin_none(max_wait_owned, max_wait_left) while True: if not len(self.waits) and not len(self.transports): self.logger('ex').debug('Exit') raise EventLoopEmpty() fds = self.nonflat_fds() # todo: SIGTERM is called here. try: polled_flags = select_helper(flatten(fds), max_wait=max_wait_step) except ConnectionAbortedError: self.logger('coabt.s').debug('%s', fds) continue elsld.step(polled_flags) self.recv(fds, polled_flags) break if max_wait_left is not None and max_wait_left == 0: break
def reg(self) -> float: logging.debug('Still waiting') if self.exit_in and time_now() > self.exit_in: raise TerminationException('Finished') return 1.
def process_queue(jobs_pending_done: SchedKeyQueue, fn: Callable[[RPCKey], Optional[float]], tol: float = 0.) -> Optional[float]: # sb = service(Broker[self.cls_req, self.cls_res], self.url_broker, group=BACKEND) now = time_now() while True: val = jobs_pending_done.peek() if val is None: return None dsec = (val.when - now).total_seconds() if dsec > tol: return dsec val = jobs_pending_done.pop() next_timeout = fn(val.key) if next_timeout: jobs_pending_done.push( SchedKey(now + timedelta(seconds=next_timeout), val.key))
def _job_new(self, jid: RPCKey, jreq: RequestType): trc('0').debug('%s %s', jid, jreq) self.jobs[jid] = JobState(jreq, time_now()) self.jobs_pending.push(jid) reset(self.push_push_assign, 0)
def push_push_assign(self): def spread(iter_obj: Iterable[Tuple[str, int]]): for wid, capa in iter_obj: for i in range(capa): yield wid def eat(obj: Deque[RPCKey]): while len(obj): try: yield obj.pop() except IndexError: return w_caps = ((wid, max( 0, self.workers[wid].load.capacity - len(self.workers_jobs[wid]))) for wid, wst in self.workers.items()) jobs_workers = zip(spread(w_caps), eat(self.jobs_pending)) for wid, jid in jobs_workers: trc('1').debug('%s %s', wid, jid) self.jobs[jid].started = time_now() self.jobs[jid].attempts += 1 self.jobs_workers[jid] = wid self.workers_jobs[wid].append(jid) self.jobs_pending_assign.push(SchedKey.now(jid)) reset(self.push_assign, 0)
def __init__(self, actor: 'Actor', group: str, chan: ELTransportRef, serde: SerdeStruct, rpcs: RPCEntrySet, horizon_each=60.): self.actor = actor self.horizon_each = horizon_each self.log_dict = RPCLogDict(time_now()) self.group = group self.chan = chan self.wait = actor.el.wait_add(ELWaitEntry(self.max_wait, self.timeout)) self.chan.push( ELPktEntry(lambda x: x.packet.type == RPCPacketType.Req, self.packet_receive, self.exception_handler)) self.serde = serde self.rpcs: RPCEntrySet = { k: v for k, v in rpcs.items() if not v.conf.exc } self.rpcs_exc: RPCEntrySet = { k: v for k, v in rpcs.items() if v.conf.exc } self.chan.push_exc(ELExcEntry(self.handle_exc))
def __call__(self, *args, **kwargs): payload = self.par.defn.serde.serialize(self.call.req, [args, kwargs]) packet = RPCPacket(self.key, RPCPacketType.Req, self.alias or self.name, payload) # the only difference between a client and a server is NONE. # the only issue would be the routing of the required packets to the required instances if self.call.conf.type == RPCType.Signalling: self.par.tran.send(RPCPacketRaw(self.par.dest, packet)) elif self.call.conf.type in [RPCType.Repliable, RPCType.Durable]: time_started = time_now() self.par.tran.push( ELPktEntry( lambda x: x.packet.key == self.key and x.packet.type == RPCPacketType.Rep, self.process_packet, partial(raise_exc_handler, should_log=False), )) try: while True: time_step = time_now() dur_passed = time_step - time_started if self.par.conf.timeout_total is not None and dur_passed.total_seconds( ) > self.par.conf.timeout_total: raise TimeoutError() self.par.tran.send(RPCPacketRaw(self.par.dest, packet)) self.par.tran.loop(max(0., self.par.conf.timeout_resend)) except RequestStopper as e: return e.return_ finally: # todo: if someone causes a TerminationException, # todo: if a server causes an EventLoopEmpty while doing this # todo: then the current transport is going to be destroyed before we reach this line if self.par.tran.is_alive(): self.par.tran.pop() else: raise NotImplementedError(self.call.conf.type)
def tick(self) -> float: if (time_now() - self.started).total_seconds() > self.timeout: raise TimeoutError('Timeout') if all([self.has_replied_b, self.has_replied_a]): raise TerminationException() return 0.5
def max_poll_regulars(self): step_time = time_now() max_poll_regulars = { k: (x - step_time).total_seconds() for k, x in self.states_regulars.items() if x is not None } return max_poll_regulars
def reset(self, name: str, new_val: Optional[float]): assert isinstance(new_val, (float, int, None.__class__)), new_val step_time = time_now() if new_val is not None: new_val = step_time + timedelta(seconds=new_val) self.states_regulars[name] = new_val
def get_horizon_wait(self) -> float: step_time = time_now() next_horizon: datetime = self.log_dict.horizon + timedelta( seconds=self.horizon_each) * 2 max_poll_horizon = (next_horizon - step_time).total_seconds() return max_poll_horizon
def _job_done(self, jid: RPCKey, jres: ResponseType): self.dones += 1 self._job_clean(jid) self.jobs[jid].finished = time_now() self.jobs[jid].res = jres self.jobs_pending_flush.push(SchedKey.now(jid)) reset(self.push_flush, 0)
def get(self, now=None) -> timedelta: if now is None: now = time_now() elapsed = now - self.started if self.max and elapsed.total_seconds() > self.max: raise TimeoutError() return elapsed
def wait_items(waiting, max_wait=40): wait_till = time_now() + timedelta(seconds=max_wait) waiting = list(waiting) while wait_till > time_now() and len(waiting): to_remove = [] for x in waiting: try: x.wait(0) to_remove.append(x) except multiprocessing.context.TimeoutError: pass except subprocess.TimeoutExpired: pass for x in to_remove: waiting.remove(x) sleep(0.03) if len(waiting) and wait_till > time_now(): raise TimeoutError(f'{waiting}')
def __getitem__(self, item): now = time_now() if item in self.values: item_rec = self.values[item] if item_rec.ts > now - self.max_timeout: return item_rec.v else: return self.hidrate_fn(self, item) return self.hidrate_fn(self, item)
def wait_all(*waiting, max_wait=5.) -> List[int]: wait_till = time_now() + timedelta(seconds=max_wait) waiting = {i: v for i, v in enumerate(waiting)} r: Dict[int, int] = {} while wait_till > time_now() and len(waiting): for i, x in list(waiting.items()): try: r[i] = x.wait(0) del waiting[i] except multiprocessing.context.TimeoutError: pass except subprocess.TimeoutExpired: pass sleep(0.03) if len(waiting): raise TimeoutError(f'{waiting}') return [v for _, v in sorted(r.items())]
def test_rpclogdict(self): cr = time_now() x = RPCLogDict(cr) with self.subTest('a'): with self.assertRaises(HorizonPassedError): x[RPCKey(time_now() - timedelta(seconds=10))] = False kv = RPCKey() with self.subTest('b'): val = True x[kv] = val self.assertEqual(x[kv], val) with self.subTest('c'): x.set_horizon(time_now()) with self.assertRaises(HorizonPassedError): x[kv]
def exec(self, __origin, __fn: Callable, *args, **kwargs): is_ok = False r = None idx = context_push(self) t_start = time_now() log_tr_exec_in.debug('[%s %s] Name=%s %s %s %s %s', idx, __origin, __fn, is_ok, args, kwargs, r) try: r = __fn(*args, **kwargs) is_ok = True return r finally: t_end = time_now() secs = (t_start - t_end).total_seconds() if secs > 0.01: log_tr_exec_out.error('[%s %s], Name=%s (%s)', idx, __origin, __fn, secs) log_tr_exec_out.debug('[%s %s] Name=%s %s %s %s %s', idx, __origin, __fn, is_ok, args, kwargs, r) context_pop(idx)
def __init__( self, actor: 'Actor', el: EventLoop, regulars: RegularEntrySet, ): # if we integrate max_waits into the RPCTransportStack self.actor = actor self.wait = el.wait_add( ELWaitEntry(self.max_wait, self.timeout, self.exception_handler)) self.regulars = regulars self.states_regulars: Dict[str, Optional[datetime]] = { k: time_now() + timedelta(seconds=x.conf.initial) if x.conf.initial is not None else None for k, x in self.regulars.items() }
def timeout(self): # a regular does not know about the transports available to it! max_poll_regulars = self.max_poll_regulars for name, reg_def in ((k, self.regulars[k]) for k, v in max_poll_regulars.items() if v is not None and v <= 0.): with self.exc_ctx(reg_def): self.states_regulars[name] = None x: float = self.actor.ctx().exec('reg', reg_def.fn) if x is not None and x < 0: trc().error('%s (%s < 0)', name, x) if x is not None: self.states_regulars[name] = time_now() + timedelta( seconds=x) else: self.states_regulars[name] = None
def bk_done(self, res: ResponseType): wid = sender() if wid not in self.workers_jobs: trc('1').error('%s', wid) return jid = self.workers_jobs[wid] if jid not in self.jobs_workers: trc('1').error('%s', jid) return self.jobs_res[jid] = res self.jobs_pending_done.push(SchedKey(time_now(), jid)) reset(self.push_done, 0) wid = self.jobs_workers[jid] del self.jobs_workers[jid] del self.workers_jobs[wid] self.load.occupied -= 1 self.workers_free.push(wid)
def __setitem__(self, key, value): self.values[key] = CachedDictRecord(time_now(), value)
def timeout(self): self.log_dict.set_horizon(time_now() - timedelta(seconds=self.horizon_each))
def __init__(self, cls: Type[T], timeout=20.): self.cls = cls self.has_replied_a = False self.has_replied_b = False self.started = time_now() self.timeout = timeout
def now(cls, key: RPCKey): return SchedKey(time_now(), key)
def new(cls): return RPCKey(time_now(), uuid4())
def __str__(self): now = time_now() total = self.when - now return f'{self.__class__.__name__}(Seconds={total.total_seconds():0.2f}s)'
def finished_a(self, jid: RPCKey, jres: Response): self.exit_in = time_now() + timedelta(seconds=0.5) trc('0').warning('Finished %s', self.exit_in) self._ack(jid)
def now(cls): return MutableDateTime(time_now())