def check_pending_actions(self):
expired, remaining = partition(lambda E: E[0].expired(),
self._pending_actions)
for each in expired:
each[1]()
self._pending_actions = remaining
return min([E[0] for E in self._pending_actions] + [ExpirationTimer(None)])
def check_pending_actions(self):
expired, remaining = partition(lambda E: E[0].expired(),
self._pending_actions)
for each in expired:
each[1]()
self._pending_actions = remaining
return min([E[0]
for E in self._pending_actions] + [ExpirationTimer(None)])
def check_pending_actions(self, current_time):
expired, rem = partition(lambda E: E[0].view(current_time).expired(),
self._pending_actions)
for each in expired:
each[1]()
self._pending_actions = rem
return min([E[0]
for E in self._pending_actions] + [ExpirationTimer(None)])
def _complete_expired_intents(self):
with self._aTB_lock:
expiredTX, validTX = partition(lambda i: i.expired(),
self._aTB_queuedPendingTransmits,
deque)
self._aTB_queuedPendingTransmits = validTX
rlen = len(validTX)
for each in expiredTX:
thesplog('TX intent %s timed out', each, level=logging.WARNING)
each.tx_done(SendStatus.Failed)
return rlen, bool(expiredTX)
def _complete_expired_intents(self):
with self._aTB_lock:
expiredTX, validTX = partition(lambda i: i.expired(),
self._aTB_queuedPendingTransmits,
deque)
self._aTB_queuedPendingTransmits = validTX
rlen = len(validTX)
for each in expiredTX:
thesplog('TX intent %s timed out', each, level=logging.WARNING)
each.tx_done(SendStatus.Failed)
return rlen, bool(expiredTX)
def childDied(self, signum, frame):
logproc = getattr(self, 'asLogProc', None)
if logproc and not self._checkChildLiveness(logproc):
# Logger has died; need to start another
if not hasattr(self, '_exiting'):
_startLogger(self.transport.__class__, self.transport, self, self.capabilities, self.logdefs)
# Signal handler for SIGCHLD; figure out which child and synthesize a ChildActorExited to handle it
self._child_procs, dead = partition(self._checkChildLiveness, getattr(self, '_child_procs', []))
for each in dead:
addr = getattr(each, 'childRealAddr', each.childAddr)
self.transport.scheduleTransmit(None, TransmitIntent(self.transport.myAddress,
ChildActorExited(addr)))
def deadAddress(self, addressManager, childAddr):
canceli, continuei = partition(lambda i: i.targetAddr == childAddr,
self._transmitIntents)
self._transmitIntents = continuei
for each in canceli:
if hasattr(each, 'socket'):
each.socket.close()
delattr(each, 'socket')
each.result = SendStatus.DeadTarget
each.completionCallback()
# No need to clean up self._incomingSockets entries: they will timeout naturally
super(TCPTransport, self).deadAddress(addressManager, childAddr)
def deadAddress(self, addressManager, childAddr):
canceli, continuei = partition(lambda i: i.targetAddr == childAddr,
self._transmitIntents)
self._transmitIntents = continuei
for each in canceli:
if hasattr(each, 'socket'):
each.socket.close()
delattr(each, 'socket')
each.result = SendStatus.DeadTarget
each.completionCallback()
# No need to clean up self._incomingSockets entries: they will timeout naturally
super(TCPTransport, self).deadAddress(addressManager, childAddr)
def deadAddress(self, addressManager, childAddr):
if childAddr in self._queues:
# Can no longer send to this Queue object. Delete the
# entry; this will cause forwarding of messages, although
# the addressManager is also aware of the dead address and
# will cause DeadEnvelope forwarding. Deleting here
# prevents hanging on queue full to dead children.
del self._queues[childAddr]
deadfwd, okfwd = ([],[]) if False else \
partition(lambda i: i[0] == childAddr or i[1] == childAddr,
self._fwdvia.items())
if deadfwd:
self._fwdvia = dict(okfwd)
super(MultiprocessQueueTransport, self).deadAddress(addressManager, childAddr)
def deadAddress(self, addressManager, childAddr):
# Can no longer send to this Queue object. Delete the
# entry; this will cause forwarding of messages, although
# the addressManager is also aware of the dead address and
# will cause DeadEnvelope forwarding. Deleting here
# prevents hanging on queue full to dead children.
self._queues.rmv(childAddr)
deadfwd, okfwd = ([],[]) if False else \
partition(lambda i: i[0] == childAddr or i[1] == childAddr,
self._fwdvia.items())
if deadfwd:
self._fwdvia = AssocList()
for A,AQ in okfwd:
self._fwdvia.add(A,AQ)
super(MultiprocessQueueTransport, self).deadAddress(addressManager, childAddr)
def childDied(self, signum, frame):
logproc = getattr(self, 'asLogProc', None)
if logproc and not self._checkChildLiveness(logproc):
# Logger has died; need to start another
if not hasattr(self, '_exiting'):
_startLogger(self.transport.__class__, self.transport, self, self.capabilities, self.logdefs)
# Signal handler for SIGCHLD; figure out which child and synthesize a ChildActorExited to handle it
self._child_procs, dead = partition(self._checkChildLiveness, getattr(self, '_child_procs', []))
for each in dead:
addr = getattr(each, 'childRealAddr', each.childAddr)
try:
self.transport.scheduleTransmit(None, TransmitIntent(self.transport.myAddress,
ChildActorExited(addr)))
except CannotPickleAddress:
thesplog('child %s is dead but cannot translate address to properly handle it',
addr, level=logging.ERROR)
def childDied(self):
logproc = getattr(self, 'asLogProc', None)
if logproc and not self._checkChildLiveness(logproc):
# Logger has died; need to start another
if not hasattr(self, '_exiting'):
_startLogger(self.transport.__class__, self.transport, self, self.capabilities, self.logdefs,
self.mpcontext)
# Signal handler for SIGCHLD; figure out which child and synthesize a ChildActorExited to handle it
self._child_procs, dead = partition(self._checkChildLiveness, getattr(self, '_child_procs', []))
for each in dead:
addr = getattr(each, 'childRealAddr', each.childAddr)
try:
self.transport.scheduleTransmit(None, TransmitIntent(self.transport.myAddress,
ChildActorExited(addr)))
except CannotPickleAddress:
thesplog('child %s is dead but cannot translate address to properly handle it',
addr, level=logging.ERROR)
return True # keep going
def _keepIf(self, keepFunc):
requeues, removes = partition(keepFunc, self._hysteresis_queue)
self._hysteresis_queue = requeues
return removes
def _runWithExpiry(self, incomingHandler):
xmitOnly = incomingHandler == TransmitOnly or \
isinstance(incomingHandler, TransmitOnly)
if hasattr(self, '_aborting_run'): delattr(self, '_aborting_run')
while not self.run_time.expired() and \
(not hasattr(self, '_aborting_run') or
(self._aborting_run and self._transmitIntents)):
if xmitOnly:
if not self._transmitIntents:
return 0
else:
while self._incomingEnvelopes:
rEnv = self._incomingEnvelopes.pop(0)
if incomingHandler is None:
return rEnv
if not incomingHandler(rEnv):
return None
wsend, wrecv = fmap(TCPTransport._socketFile,
partition(TCPTransport._waitForSendable,
filter(lambda T: not T.backoffPause(),
self._transmitIntents)))
wrecv = list(filter(None, wrecv))
wsend = list(filter(None, wsend))
wrecv.extend(list(filter(None,
[TCPTransport._socketFile(I)
for I in self._incomingSockets
if not I.backoffPause()])))
delays = list([R for R in [self.run_time.remaining()] +
[T.delay() for T in self._transmitIntents] +
[T.delay() for T in self._incomingSockets]
if R is not None])
delay = timePeriodSeconds(min(delays)) if delays else None
if not hasattr(self, '_aborting_run') and not xmitOnly:
wrecv.extend([self.socket.fileno()])
# rrecv, rsend, _ign2 = select.select(wrecv, wsend, [], delay)
try:
rrecv, rsend, _ign2 = select.select(wrecv, wsend, set(wsend+wrecv),
delay)
except ValueError as ex:
thesplog('ValueError on select(#%d: %s, #%d: %s, #%d: %s, %s)',
len(wrecv), wrecv, len(wsend), wsend,
len(set(wsend + wrecv)), set(wsend + wrecv),
delay, level=logging.ERROR)
raise
except select.error as ex:
if ex.args[0] in (errno.EINVAL, # probably a change in descriptors
errno.EINTR,
):
thesplog('select retry on %s', ex, level=logging.ERROR)
continue
raise
if _ign2:
thesplog('WHOA... something else to do for sockets: %s',
_ign2, level=logging.WARNING)
origPendingSends = len(self._transmitIntents)
# Handle newly sendable data
for eachs in rsend:
self._transmitIntents = [I for I in self._transmitIntents
if self._nextTransmitStepCheck(I, eachs)]
# Handle newly receivable data
for each in rrecv:
if each == self.socket.fileno():
self._acceptNewIncoming()
continue
self._incomingSockets = [S for S in self._incomingSockets
if self._handlePossibleIncoming(S, each)]
self._transmitIntents = [I for I in self._transmitIntents
if self._nextTransmitStepCheck(I, each)]
# Handle timeouts
self._transmitIntents = [I for I in self._transmitIntents
if self._nextTransmitStepCheck(I, -1)]
self._incomingSockets = [S for S in self._incomingSockets
if self._handlePossibleIncoming(S, -1)]
# Check if it's time to quit
if [] == rrecv and [] == rsend:
if [] == _ign2 and self.run_time.expired():
# Timeout, give up
return None
continue
if xmitOnly:
remXmits = len(self._transmitIntents)
if origPendingSends > remXmits or remXmits == 0:
return remXmits
# Handle queued internal "received" data
if not xmitOnly:
while self._incomingEnvelopes:
rEnv = self._incomingEnvelopes.pop(0)
if incomingHandler is None:
return rEnv
if not incomingHandler(rEnv):
return None
return None
def _pop_expired_wakeups(self):
exp, self._wakeUps = partition(lambda E: E[1].expired(), self._wakeUps)
return exp
def _check_tx_queue_length(self):
while self._complete_expired_intents():
pass
with self._aTB_lock:
validTX, expiredTX = partition(lambda i: i.expired(),
self._aTB_queuedPendingTransmits)
def _runWithExpiry(self, incomingHandler):
xmitOnly = incomingHandler == TransmitOnly or \
isinstance(incomingHandler, TransmitOnly)
if hasattr(self, '_aborting_run'): delattr(self, '_aborting_run')
while not self.run_time.expired() and \
(not hasattr(self, '_aborting_run') or
(self._aborting_run and self._transmitIntents)):
if xmitOnly:
if not self._transmitIntents:
return 0
else:
while self._incomingEnvelopes:
rEnv = self._incomingEnvelopes.pop(0)
if incomingHandler is None:
return rEnv
if not incomingHandler(rEnv):
return None
wsend, wrecv = fmap(
TCPTransport._socketFile,
partition(
TCPTransport._waitForSendable,
filter(lambda T: not T.backoffPause(),
self._transmitIntents)))
wrecv = list(filter(None, wrecv))
wsend = list(filter(None, wsend))
wrecv.extend(
list(
filter(None, [
TCPTransport._socketFile(I)
for I in self._incomingSockets if not I.backoffPause()
])))
delays = list([
R for R in [self.run_time.remaining()] +
[T.delay() for T in self._transmitIntents] +
[T.delay() for T in self._incomingSockets] if R is not None
])
delay = timePeriodSeconds(min(delays)) if delays else None
if not hasattr(self, '_aborting_run') and not xmitOnly:
wrecv.extend([self.socket.fileno()])
# rrecv, rsend, _ign2 = select.select(wrecv, wsend, [], delay)
try:
rrecv, rsend, _ign2 = select.select(wrecv, wsend,
set(wsend + wrecv), delay)
except ValueError as ex:
thesplog('ValueError on select(#%d: %s, #%d: %s, #%d: %s, %s)',
len(wrecv),
wrecv,
len(wsend),
wsend,
len(set(wsend + wrecv)),
set(wsend + wrecv),
delay,
level=logging.ERROR)
raise
except select.error as ex:
if ex.args[0] in (
errno.EINVAL, # probably a change in descriptors
errno.EINTR,
):
thesplog('select retry on %s', ex, level=logging.ERROR)
continue
raise
if _ign2:
thesplog('WHOA... something else to do for sockets: %s',
_ign2,
level=logging.WARNING)
origPendingSends = len(self._transmitIntents)
# Handle newly sendable data
for eachs in rsend:
self._transmitIntents = [
I for I in self._transmitIntents
if self._nextTransmitStepCheck(I, eachs)
]
# Handle newly receivable data
for each in rrecv:
if each == self.socket.fileno():
self._acceptNewIncoming()
continue
self._incomingSockets = [
S for S in self._incomingSockets
if self._handlePossibleIncoming(S, each)
]
self._transmitIntents = [
I for I in self._transmitIntents
if self._nextTransmitStepCheck(I, each)
]
# Handle timeouts
self._transmitIntents = [
I for I in self._transmitIntents
if self._nextTransmitStepCheck(I, -1)
]
self._incomingSockets = [
S for S in self._incomingSockets
if self._handlePossibleIncoming(S, -1)
]
# Check if it's time to quit
if [] == rrecv and [] == rsend:
if [] == _ign2 and self.run_time.expired():
# Timeout, give up
return None
continue
if xmitOnly:
remXmits = len(self._transmitIntents)
if origPendingSends > remXmits or remXmits == 0:
return remXmits
# Handle queued internal "received" data
if not xmitOnly:
while self._incomingEnvelopes:
rEnv = self._incomingEnvelopes.pop(0)
if incomingHandler is None:
return rEnv
if not incomingHandler(rEnv):
return None
return None
def _pop_expired_wakeups(self, ct):
exp, self._wakeUps = partition(lambda E: E[1].view(ct).expired(),
self._wakeUps)
return exp
def _keepIf(self, keepFunc):
requeues, removes = partition(keepFunc, self._hysteresis_queue)
self._hysteresis_queue = requeues
return removes