def step(self, polled_flags):
if any(polled_flags) and polled_flags == self.flgs:
self.flgs_ctr += 1
elif self.flgs is None:
self.flgs = polled_flags
self.flgs_ctr = 0
self.time = time_now()
else:
self.flgs = polled_flags
self.flgs_ctr = 0
self.time = time_now()
fa = any(self.flgs) and self.flgs_ctr > self.max_ctr
vb = fa and ((time_now() - self.time).total_seconds() / self.flgs_ctr)
fb = fa and vb < self.max_iter_secs
if fa and fb:
lgr = self.par.logger('spinlock')
lgr.warning('%s %s %s', self.flgs_ctr, self.flgs, vb)
lgr.warning('%s', self.par.nonflat_fds())
lgr.warning('%s', self.par.transports)
lgr.warning('%s', self.par.waits)
_log_traceback(lgr, 'We are in a spinlock')
time.sleep(self.sleep_secs)
def loop(self, max_wait: Optional[float] = None):
ts = time_now()
def bin_none(a, b, fn=min):
if a is None:
return b
elif b is None:
return a
else:
return fn(a, b)
elsld = ELSpinLockDetector(self)
while True:
running_for = (time_now() - ts).total_seconds()
max_wait_left = None if max_wait is None else max(
max_wait - running_for, 0)
max_wait_owned: Optional[float] = None
max_waits_curr = self.max_waits
for k, x in max_waits_curr.items():
max_wait_owned = bin_none(max_wait_owned, x)
if x is not None and x <= 0.:
wait = self.waits.get(k)
if wait is None:
continue
with exc_handler(wait.exc):
wait.processor()
max_wait_step = bin_none(max_wait_owned, max_wait_left)
while True:
if not len(self.waits) and not len(self.transports):
self.logger('ex').debug('Exit')
raise EventLoopEmpty()
fds = self.nonflat_fds()
# todo: SIGTERM is called here.
try:
polled_flags = select_helper(flatten(fds),
max_wait=max_wait_step)
except ConnectionAbortedError:
self.logger('coabt.s').debug('%s', fds)
continue
elsld.step(polled_flags)
self.recv(fds, polled_flags)
break
if max_wait_left is not None and max_wait_left == 0:
break
def reg(self) -> float:
logging.debug('Still waiting')
if self.exit_in and time_now() > self.exit_in:
raise TerminationException('Finished')
return 1.
def process_queue(jobs_pending_done: SchedKeyQueue,
fn: Callable[[RPCKey], Optional[float]],
tol: float = 0.) -> Optional[float]:
# sb = service(Broker[self.cls_req, self.cls_res], self.url_broker, group=BACKEND)
now = time_now()
while True:
val = jobs_pending_done.peek()
if val is None:
return None
dsec = (val.when - now).total_seconds()
if dsec > tol:
return dsec
val = jobs_pending_done.pop()
next_timeout = fn(val.key)
if next_timeout:
jobs_pending_done.push(
SchedKey(now + timedelta(seconds=next_timeout), val.key))
def _job_new(self, jid: RPCKey, jreq: RequestType):
trc('0').debug('%s %s', jid, jreq)
self.jobs[jid] = JobState(jreq, time_now())
self.jobs_pending.push(jid)
reset(self.push_push_assign, 0)
def push_push_assign(self):
def spread(iter_obj: Iterable[Tuple[str, int]]):
for wid, capa in iter_obj:
for i in range(capa):
yield wid
def eat(obj: Deque[RPCKey]):
while len(obj):
try:
yield obj.pop()
except IndexError:
return
w_caps = ((wid,
max(
0, self.workers[wid].load.capacity -
len(self.workers_jobs[wid])))
for wid, wst in self.workers.items())
jobs_workers = zip(spread(w_caps), eat(self.jobs_pending))
for wid, jid in jobs_workers:
trc('1').debug('%s %s', wid, jid)
self.jobs[jid].started = time_now()
self.jobs[jid].attempts += 1
self.jobs_workers[jid] = wid
self.workers_jobs[wid].append(jid)
self.jobs_pending_assign.push(SchedKey.now(jid))
reset(self.push_assign, 0)
def __init__(self,
actor: 'Actor',
group: str,
chan: ELTransportRef,
serde: SerdeStruct,
rpcs: RPCEntrySet,
horizon_each=60.):
self.actor = actor
self.horizon_each = horizon_each
self.log_dict = RPCLogDict(time_now())
self.group = group
self.chan = chan
self.wait = actor.el.wait_add(ELWaitEntry(self.max_wait, self.timeout))
self.chan.push(
ELPktEntry(lambda x: x.packet.type == RPCPacketType.Req,
self.packet_receive, self.exception_handler))
self.serde = serde
self.rpcs: RPCEntrySet = {
k: v
for k, v in rpcs.items() if not v.conf.exc
}
self.rpcs_exc: RPCEntrySet = {
k: v
for k, v in rpcs.items() if v.conf.exc
}
self.chan.push_exc(ELExcEntry(self.handle_exc))
def __call__(self, *args, **kwargs):
payload = self.par.defn.serde.serialize(self.call.req, [args, kwargs])
packet = RPCPacket(self.key, RPCPacketType.Req, self.alias
or self.name, payload)
# the only difference between a client and a server is NONE.
# the only issue would be the routing of the required packets to the required instances
if self.call.conf.type == RPCType.Signalling:
self.par.tran.send(RPCPacketRaw(self.par.dest, packet))
elif self.call.conf.type in [RPCType.Repliable, RPCType.Durable]:
time_started = time_now()
self.par.tran.push(
ELPktEntry(
lambda x: x.packet.key == self.key and x.packet.type ==
RPCPacketType.Rep,
self.process_packet,
partial(raise_exc_handler, should_log=False),
))
try:
while True:
time_step = time_now()
dur_passed = time_step - time_started
if self.par.conf.timeout_total is not None and dur_passed.total_seconds(
) > self.par.conf.timeout_total:
raise TimeoutError()
self.par.tran.send(RPCPacketRaw(self.par.dest, packet))
self.par.tran.loop(max(0., self.par.conf.timeout_resend))
except RequestStopper as e:
return e.return_
finally:
# todo: if someone causes a TerminationException,
# todo: if a server causes an EventLoopEmpty while doing this
# todo: then the current transport is going to be destroyed before we reach this line
if self.par.tran.is_alive():
self.par.tran.pop()
else:
raise NotImplementedError(self.call.conf.type)
def tick(self) -> float:
if (time_now() - self.started).total_seconds() > self.timeout:
raise TimeoutError('Timeout')
if all([self.has_replied_b, self.has_replied_a]):
raise TerminationException()
return 0.5
def max_poll_regulars(self):
step_time = time_now()
max_poll_regulars = {
k: (x - step_time).total_seconds()
for k, x in self.states_regulars.items() if x is not None
}
return max_poll_regulars
def reset(self, name: str, new_val: Optional[float]):
assert isinstance(new_val, (float, int, None.__class__)), new_val
step_time = time_now()
if new_val is not None:
new_val = step_time + timedelta(seconds=new_val)
self.states_regulars[name] = new_val
def get_horizon_wait(self) -> float:
step_time = time_now()
next_horizon: datetime = self.log_dict.horizon + timedelta(
seconds=self.horizon_each) * 2
max_poll_horizon = (next_horizon - step_time).total_seconds()
return max_poll_horizon
def _job_done(self, jid: RPCKey, jres: ResponseType):
self.dones += 1
self._job_clean(jid)
self.jobs[jid].finished = time_now()
self.jobs[jid].res = jres
self.jobs_pending_flush.push(SchedKey.now(jid))
reset(self.push_flush, 0)
def get(self, now=None) -> timedelta:
if now is None:
now = time_now()
elapsed = now - self.started
if self.max and elapsed.total_seconds() > self.max:
raise TimeoutError()
return elapsed
def wait_items(waiting, max_wait=40):
wait_till = time_now() + timedelta(seconds=max_wait)
waiting = list(waiting)
while wait_till > time_now() and len(waiting):
to_remove = []
for x in waiting:
try:
x.wait(0)
to_remove.append(x)
except multiprocessing.context.TimeoutError:
pass
except subprocess.TimeoutExpired:
pass
for x in to_remove:
waiting.remove(x)
sleep(0.03)
if len(waiting) and wait_till > time_now():
raise TimeoutError(f'{waiting}')
def __getitem__(self, item):
now = time_now()
if item in self.values:
item_rec = self.values[item]
if item_rec.ts > now - self.max_timeout:
return item_rec.v
else:
return self.hidrate_fn(self, item)
return self.hidrate_fn(self, item)
def wait_all(*waiting, max_wait=5.) -> List[int]:
wait_till = time_now() + timedelta(seconds=max_wait)
waiting = {i: v for i, v in enumerate(waiting)}
r: Dict[int, int] = {}
while wait_till > time_now() and len(waiting):
for i, x in list(waiting.items()):
try:
r[i] = x.wait(0)
del waiting[i]
except multiprocessing.context.TimeoutError:
pass
except subprocess.TimeoutExpired:
pass
sleep(0.03)
if len(waiting):
raise TimeoutError(f'{waiting}')
return [v for _, v in sorted(r.items())]
def test_rpclogdict(self):
cr = time_now()
x = RPCLogDict(cr)
with self.subTest('a'):
with self.assertRaises(HorizonPassedError):
x[RPCKey(time_now() - timedelta(seconds=10))] = False
kv = RPCKey()
with self.subTest('b'):
val = True
x[kv] = val
self.assertEqual(x[kv], val)
with self.subTest('c'):
x.set_horizon(time_now())
with self.assertRaises(HorizonPassedError):
x[kv]
def exec(self, __origin, __fn: Callable, *args, **kwargs):
is_ok = False
r = None
idx = context_push(self)
t_start = time_now()
log_tr_exec_in.debug('[%s %s] Name=%s %s %s %s %s', idx, __origin, __fn, is_ok, args, kwargs, r)
try:
r = __fn(*args, **kwargs)
is_ok = True
return r
finally:
t_end = time_now()
secs = (t_start - t_end).total_seconds()
if secs > 0.01:
log_tr_exec_out.error('[%s %s], Name=%s (%s)', idx, __origin, __fn, secs)
log_tr_exec_out.debug('[%s %s] Name=%s %s %s %s %s', idx, __origin, __fn, is_ok, args, kwargs, r)
context_pop(idx)
def __init__(
self,
actor: 'Actor',
el: EventLoop,
regulars: RegularEntrySet,
):
# if we integrate max_waits into the RPCTransportStack
self.actor = actor
self.wait = el.wait_add(
ELWaitEntry(self.max_wait, self.timeout, self.exception_handler))
self.regulars = regulars
self.states_regulars: Dict[str, Optional[datetime]] = {
k: time_now() + timedelta(seconds=x.conf.initial)
if x.conf.initial is not None else None
for k, x in self.regulars.items()
}
def timeout(self):
# a regular does not know about the transports available to it!
max_poll_regulars = self.max_poll_regulars
for name, reg_def in ((k, self.regulars[k])
for k, v in max_poll_regulars.items()
if v is not None and v <= 0.):
with self.exc_ctx(reg_def):
self.states_regulars[name] = None
x: float = self.actor.ctx().exec('reg', reg_def.fn)
if x is not None and x < 0:
trc().error('%s (%s < 0)', name, x)
if x is not None:
self.states_regulars[name] = time_now() + timedelta(
seconds=x)
else:
self.states_regulars[name] = None
def bk_done(self, res: ResponseType):
wid = sender()
if wid not in self.workers_jobs:
trc('1').error('%s', wid)
return
jid = self.workers_jobs[wid]
if jid not in self.jobs_workers:
trc('1').error('%s', jid)
return
self.jobs_res[jid] = res
self.jobs_pending_done.push(SchedKey(time_now(), jid))
reset(self.push_done, 0)
wid = self.jobs_workers[jid]
del self.jobs_workers[jid]
del self.workers_jobs[wid]
self.load.occupied -= 1
self.workers_free.push(wid)
def __setitem__(self, key, value):
self.values[key] = CachedDictRecord(time_now(), value)
def timeout(self):
self.log_dict.set_horizon(time_now() -
timedelta(seconds=self.horizon_each))
def __init__(self, cls: Type[T], timeout=20.):
self.cls = cls
self.has_replied_a = False
self.has_replied_b = False
self.started = time_now()
self.timeout = timeout
def now(cls, key: RPCKey):
return SchedKey(time_now(), key)
def new(cls):
return RPCKey(time_now(), uuid4())
def __str__(self):
now = time_now()
total = self.when - now
return f'{self.__class__.__name__}(Seconds={total.total_seconds():0.2f}s)'
def finished_a(self, jid: RPCKey, jres: Response):
self.exit_in = time_now() + timedelta(seconds=0.5)
trc('0').warning('Finished %s', self.exit_in)
self._ack(jid)
def now(cls):
return MutableDateTime(time_now())
