def testNonZeroRemainingSeconds(self):
et = ExpirationTimer(timedelta(milliseconds=10))
with et as c:
assert 0.0 < c.remainingSeconds()
assert 0.0101 > c.remainingSeconds()
sleep(et.view().remainingSeconds())
assert 0.0 == et.view().remainingSeconds()
def tell(self, anActor, msg):
attemptLimit = ExpirationTimer(MAX_TELL_PERIOD)
# transport may not use sockets, but this helps error handling
# in case it does.
import socket
for attempt in range(5000):
try:
txwatch = self._tx_to_actor(anActor, msg)
while not attemptLimit.expired():
if not self._run_transport(attemptLimit.remaining(),
txonly=True):
# all transmits completed
return
if txwatch.failed:
raise ActorSystemFailure(
'Error sending to %s: %s' % (str(anActor),
str(txwatch.failure)))
raise ActorSystemRequestTimeout(
'Unable to send to %s within %s' %
(str(anActor), str(MAX_TELL_PERIOD)))
except socket.error as ex:
import errno
if errno.EMFILE == ex.errno:
import time
time.sleep(0.1)
else:
raise
def testLessThanArbitrary(self):
et1 = ExpirationTimer(None)
et2 = ExpirationTimer(0)
et3 = ExpirationTimer(10)
with pytest.raises(AttributeError):
assert not et1 < 0
with pytest.raises(AttributeError):
assert not et2 < 0
with pytest.raises(AttributeError):
assert not et3 < 0
with pytest.raises(AttributeError):
assert not et1 < 'hi'
with pytest.raises(AttributeError):
assert not et2 < 'hi'
with pytest.raises(AttributeError):
assert not et3 < 'hi'
with pytest.raises(AttributeError):
assert not et1 < object()
with pytest.raises(AttributeError):
assert not et2 < object()
with pytest.raises(AttributeError):
assert not et3 < object()
def testNonZeroIsFalse(self):
et = ExpirationTimer(timedelta(milliseconds=10))
assert not et
assert not bool(et)
sleep(et.remainingSeconds())
assert et
assert bool(et)
def testGreaterThanArbitrary(self):
et1 = ExpirationTimer(None)
et2 = ExpirationTimer(0)
et3 = ExpirationTimer(10)
with pytest.raises(AttributeError):
assert not et1 > 0
with pytest.raises(AttributeError):
assert not et2 > 0
with pytest.raises(AttributeError):
assert not et3 > 0
with pytest.raises(AttributeError):
assert not et1 > 'hi'
with pytest.raises(AttributeError):
assert not et2 > 'hi'
with pytest.raises(AttributeError):
assert not et3 > 'hi'
with pytest.raises(AttributeError):
assert not et1 > object()
with pytest.raises(AttributeError):
assert not et2 > object()
with pytest.raises(AttributeError):
assert not et3 > object()
def shutdown(self):
thesplog('ActorSystem shutdown requested.', level=logging.INFO)
time_to_quit = ExpirationTimer(MAX_SYSTEM_SHUTDOWN_DELAY)
txwatch = self._tx_to_admin(SystemShutdown())
while not time_to_quit.expired():
response = self._run_transport(time_to_quit.remaining())
if txwatch.failed:
thesplog('Could not send shutdown request to Admin'
'; aborting but not necessarily stopped',
level=logging.WARNING)
return
if isinstance(response, ReceiveEnvelope):
if isinstance(response.message, SystemShutdownCompleted):
break
else:
thesplog('Expected shutdown completed message, got: %s', response.message,
level=logging.WARNING)
elif isinstance(response, (Thespian__Run_Expired,
Thespian__Run_Terminated,
Thespian__Run_Expired)):
break
else:
thesplog('No response to Admin shutdown request; Actor system not completely shutdown',
level=logging.ERROR)
self.transport.close()
thesplog('ActorSystem shutdown complete.')
def testArbitraryEquality(self):
et1 = ExpirationTimer(None)
et2 = ExpirationTimer(0)
et3 = ExpirationTimer(10)
assert not et1 == None
assert not et2 == None
assert not et3 == None
assert et1 != None
assert et2 != None
assert et3 != None
assert not et1 == 0
assert not et2 == 0
assert not et3 == 0
assert et1 != 0
assert et2 != 0
assert et3 != 0
assert not et1 == 'hi'
assert not et2 == 'hi'
assert not et3 == 'hi'
assert et1 != 'hi'
assert et2 != 'hi'
assert et3 != 'hi'
assert not et1 == object()
assert not et2 == object()
assert not et3 == object()
assert et1 != object()
assert et2 != object()
assert et3 != object()
def ask(self, anActor, msg, timeout):
txwatch = self._tx_to_actor(anActor, msg) # KWQ: pass timeout on tx??
askLimit = ExpirationTimer(toTimeDeltaOrNone(timeout))
while not askLimit.expired():
response = self._run_transport(askLimit.remaining())
if txwatch.failed:
if txwatch.failure in [SendStatus.DeadTarget,
SendStatus.Failed,
SendStatus.NotSent]:
# Silent failure; not all transports can indicate
# this, so for conformity the Dead Letter handler is
# the intended method of handling this issue.
return None
raise ActorSystemFailure('Transmit of ask message to %s failed (%s)'%(
str(anActor),
str(txwatch.failure)))
if not isinstance(response, ReceiveEnvelope):
# Timed out or other failure, give up.
break
# Do not send miscellaneous ActorSystemMessages to the
# caller that it might not recognize. If one of those was
# recieved, loop to get another response.
if not isInternalActorSystemMessage(response.message):
return response.message
return None
def shutdown(self):
thesplog('ActorSystem shutdown requested.', level=logging.INFO)
time_to_quit = ExpirationTimer(MAX_SYSTEM_SHUTDOWN_DELAY)
txwatch = self._tx_to_admin(SystemShutdown())
while not time_to_quit.expired():
response = self._run_transport(time_to_quit.remaining())
if txwatch.failed:
thesplog(
'Could not send shutdown request to Admin'
'; aborting but not necessarily stopped',
level=logging.WARNING)
return
if isinstance(response, ReceiveEnvelope):
if isinstance(response.message, SystemShutdownCompleted):
break
else:
thesplog('Expected shutdown completed message, got: %s',
response.message,
level=logging.WARNING)
elif isinstance(response,
(Thespian__Run_Expired, Thespian__Run_Terminated,
Thespian__Run_Expired)):
break
else:
thesplog(
'No response to Admin shutdown request; Actor system not completely shutdown',
level=logging.ERROR)
self.transport.close()
thesplog('ActorSystem shutdown complete.')
def ask(self, anActor, msg, timeout):
txwatch = self._tx_to_actor(anActor, msg) # KWQ: pass timeout on tx??
askLimit = ExpirationTimer(toTimeDeltaOrNone(timeout))
while not askLimit.expired():
response = self._run_transport(askLimit.remaining())
if txwatch.failed:
if txwatch.failure in [
SendStatus.DeadTarget, SendStatus.Failed,
SendStatus.NotSent
]:
# Silent failure; not all transports can indicate
# this, so for conformity the Dead Letter handler is
# the intended method of handling this issue.
return None
raise ActorSystemFailure(
'Transmit of ask message to %s failed (%s)' %
(str(anActor), str(txwatch.failure)))
if not isinstance(response, ReceiveEnvelope):
# Timed out or other failure, give up.
break
# Do not send miscellaneous ActorSystemMessages to the
# caller that it might not recognize. If one of those was
# recieved, loop to get another response.
if not isInternalActorSystemMessage(response.message):
return response.message
return None
def tell(self, anActor, msg):
attemptLimit = ExpirationTimer(MAX_TELL_PERIOD)
# transport may not use sockets, but this helps error handling
# in case it does.
import socket
for attempt in range(5000):
try:
txwatch = self._tx_to_actor(anActor, msg)
while not attemptLimit.expired():
if not self._run_transport(attemptLimit.remaining(),
txonly=True):
# all transmits completed
return
if txwatch.failed:
raise ActorSystemFailure(
'Error sending to %s: %s' %
(str(anActor), str(txwatch.failure)))
raise ActorSystemRequestTimeout(
'Unable to send to %s within %s' %
(str(anActor), str(MAX_TELL_PERIOD)))
except socket.error as ex:
import errno
if errno.EMFILE == ex.errno:
import time
time.sleep(0.1)
else:
raise
def testLessThanNone(self):
et1 = ExpirationTimer(None)
et2 = ExpirationTimer(0)
et3 = ExpirationTimer(10)
assert not et1 < None
assert not et2 < None
assert not et3 < None
def _run_transport(self, maximumDuration=None, txonly=False,
incomingHandler=None):
# This is where multiple external threads are synchronized for
# receives. Transmits will flow down into the transmit layer
# where they are queued with thread safety, but threads
# blocking on a receive will all be lined up through this point.
max_runtime = ExpirationTimer(maximumDuration)
with self._cv:
while self._transport_runner:
self._cv.wait(max_runtime.remainingSeconds())
if max_runtime.expired():
return None
self._transport_runner = True
try:
r = Thespian__UpdateWork()
while isinstance(r, Thespian__UpdateWork):
r = self.transport.run(TransmitOnly if txonly else incomingHandler,
max_runtime.remaining())
return r
# incomingHandler callback could deadlock on this same thread; is it ever not None?
finally:
with self._cv:
self._transport_runner = False
self._cv.notify()
def run(self, incomingHandler, maximumDuration=None):
"""Core scheduling method; called by the current Actor process when
idle to await new messages (or to do background
processing).
"""
self._max_runtime = ExpirationTimer(maximumDuration)
# Always make at least one pass through to handle expired wakeups
# and queued events; otherwise a null/negative maximumDuration could
# block all processing.
firstTime = True
while firstTime or not self._max_runtime.expired():
firstTime = False
self._update_runtime()
rval = self._runWithExpiry(incomingHandler)
if rval is not None:
return rval
if not self._realizeWakeups():
# No wakeups were processed, and the inner run
# returned, so assume there's nothing to do and exit
return rval
while self._activeWakeups:
w = self._activeWakeups.pop()
if incomingHandler in (None, TransmitOnly):
return w
r = incomingHandler(w)
if not r:
return r
return None
def _run_transport(self, maximumDuration=None, txonly=False,
incomingHandler=None):
# This is where multiple external threads are synchronized for
# receives. Transmits will flow down into the transmit layer
# where they are queued with thread safety, but threads
# blocking on a receive will all be lined up through this point.
max_runtime = ExpirationTimer(maximumDuration)
with self._cv:
while self._transport_runner:
self._cv.wait(max_runtime.remainingSeconds())
if max_runtime.expired():
return None
self._transport_runner = True
try:
r = Thespian__UpdateWork()
while isinstance(r, Thespian__UpdateWork):
r = self.transport.run(TransmitOnly if txonly else incomingHandler,
max_runtime.remaining())
return r
# incomingHandler callback could deadlock on this same thread; is it ever not None?
finally:
with self._cv:
self._transport_runner = False
self._cv.notify()
def _drain_tx_queue_if_needed(self, max_delay=None):
v, _ = self._complete_expired_intents()
if v >= MAX_QUEUED_TRANSMITS and self._aTB_rx_pause_enabled:
# Try to drain our local work before accepting more
# because it looks like we're getting really behind. This
# is dangerous though, because if other Actors are having
# the same issue this can create a deadlock.
thesplog(
'Entering tx-only mode to drain excessive queue'
' (%s > %s, drain-to %s)',
v,
MAX_QUEUED_TRANSMITS,
QUEUE_TRANSMIT_UNBLOCK_THRESHOLD,
level=logging.WARNING)
finish_time = ExpirationTimer(max_delay if max_delay else None)
while v > QUEUE_TRANSMIT_UNBLOCK_THRESHOLD and not finish_time.expired(
):
if 0 == self.run(TransmitOnly, finish_time.remaining()):
thesplog(
'Exiting tx-only mode because no transport work available.'
)
break
v, _ = self._complete_expired_intents()
thesplog('Exited tx-only mode after draining excessive queue (%s)',
len(self._aTB_queuedPendingTransmits),
level=logging.WARNING)
def _runSends(self, timeout=None, stop_on_available=False):
numsends = 0
endtime = ExpirationTimer(toTimeDeltaOrNone(timeout))
while not endtime.expired():
while self._pendingSends:
numsends += 1
if self.procLimit and numsends > self.procLimit:
raise RuntimeError('Too many sends')
self._realizeWakeups()
with self._private_lock:
try:
nextmsg = self._pendingSends.pop(0)
except IndexError:
pass
else:
self._runSingleSend(nextmsg)
if stop_on_available and \
any([not isInternalActorSystemMessage(M)
for M in getattr(stop_on_available.instance,
'responses', [])]):
return
if endtime.remaining(forever=-1) == -1:
return
next_wakeup = self._next_wakeup()
if next_wakeup is None or next_wakeup > endtime:
return
time.sleep(max(0, timePeriodSeconds(next_wakeup.remaining())))
self._realizeWakeups()
def setup_convention(self, activation=False):
self._has_been_activated |= activation
rmsgs = []
# If not specified in capabilities, don't override any invites
# that may have been received.
self._conventionAddress = self._getConventionAddr(self.capabilities) or \
self._conventionAddress
leader_is_gone = (self._conventionMembers.find(self.conventionLeaderAddr) is None) \
if self.conventionLeaderAddr else True
if not self.isConventionLeader() and self.conventionLeaderAddr:
thesplog('Admin registering with Convention @ %s (%s)',
self.conventionLeaderAddr,
'first time' if leader_is_gone else 're-registering',
level=logging.INFO,
primary=True)
rmsgs.append(
HysteresisSend(self.conventionLeaderAddr,
ConventionRegister(self.myAddress,
self.capabilities,
leader_is_gone),
onSuccess=self._setupConventionCBGood,
onError=self._setupConventionCBError))
rmsgs.append(LogAggregator(self.conventionLeaderAddr))
self._conventionRegistration = ExpirationTimer(
CONVENTION_REREGISTRATION_PERIOD)
return rmsgs
def testNonZeroIsFalse(self):
et = ExpirationTimer(timedelta(milliseconds=10))
assert not et
assert not bool(et)
sleep(et.remainingSeconds())
assert et
assert bool(et)
def testNoneToUnExpiredComparison(self):
et1 = ExpirationTimer(None)
et2 = ExpirationTimer(timedelta(milliseconds=10))
assert et1 != et2
assert et2 != et1
sleep(et2.view().remainingSeconds())
assert et1 != et2
assert et2 != et1
def testExpiredToUnExpiredComparison(self):
et1 = ExpirationTimer(timedelta(microseconds=0))
et2 = ExpirationTimer(timedelta(milliseconds=10))
assert et1 != et2
assert et2 != et1
sleep(et2.remainingSeconds())
assert et1 == et2
assert et2 == et1
def testNoneComparedToZero(self):
et1 = ExpirationTimer(None)
et2 = ExpirationTimer(timedelta(days=0))
# None == forever, so it is greater than anything, although equal to itself
assert et1 > et2
assert et2 < et1
assert et1 >= et2
assert et2 <= et1
def testExpiredToUnExpiredComparison(self):
et1 = ExpirationTimer(timedelta(microseconds=0))
et2 = ExpirationTimer(timedelta(milliseconds=10))
assert et1 != et2
assert et2 != et1
sleep(et2.remainingSeconds())
assert et1 == et2
assert et2 == et1
def newPrimaryActor(self,
actorClass,
targetActorRequirements,
globalName,
sourceHash=None):
self._numPrimaries = self._numPrimaries + 1
actorClassName = '%s.%s'%(actorClass.__module__, actorClass.__name__) \
if hasattr(actorClass, '__name__') else actorClass
with closing(self.transport.external_transport_clone()) as tx_external:
response = NewActorResponse(tx_external, self.adminAddr)
tx_external.scheduleTransmit(
None,
TransmitIntent(self.adminAddr,
PendingActor(actorClassName,
None,
self._numPrimaries,
targetActorRequirements,
globalName=globalName,
sourceHash=sourceHash),
onError=response.transmit_failed))
endwait = ExpirationTimer(MAX_CHILD_ACTOR_CREATE_DELAY)
# Do not use _run_transport: the tx_external transport
# context acquired above is unique to this thread and
# should not be synchronized/restricted by other threads.
tx_external.run(response, MAX_CHILD_ACTOR_CREATE_DELAY)
# Other items might abort the transport run... like transmit
# failures on a previous ask() that itself already timed out.
while response.pending and not endwait.expired():
tx_external.run(response, MAX_CHILD_ACTOR_CREATE_DELAY)
if response.failed:
if response.failure == PendingActorResponse.ERROR_Invalid_SourceHash:
raise InvalidActorSourceHash(sourceHash)
if response.failure == PendingActorResponse.ERROR_Invalid_ActorClass:
raise InvalidActorSpecification(actorClass,
response.failure_message)
if response.failure == PendingActorResponse.ERROR_Import:
info = response.failure_message
if info:
thesplog('Actor Create Failure, Import Error: %s', info)
raise ImportError(str(actorClass) + ': ' + info)
thesplog('Actor Create Failure, Import Error')
raise ImportError(actorClass)
if response.failure == PendingActorResponse.ERROR_No_Compatible_ActorSystem:
raise NoCompatibleSystemForActor(
actorClass, 'No compatible ActorSystem could be found')
raise ActorSystemFailure(
"Could not request new Actor from Admin (%s)" %
(response.failure))
if response.actor_address:
return response.actor_address
if response.actor_address is False:
raise NoCompatibleSystemForActor(
actorClass, 'No compatible ActorSystem could be found')
raise ActorSystemRequestTimeout(
'No response received to PendingActor request to Admin'
' at %s from %s' %
(str(self.adminAddr), str(self.transport.myAddress)))
def _update_remaining_hysteresis_period(self, reset=False):
if not self._current_hysteresis:
self._hysteresis_until = ExpirationTimer(timedelta(seconds=0))
else:
if reset or not self._hysteresis_until:
self._hysteresis_until = ExpirationTimer(self._current_hysteresis)
else:
self._hysteresis_until = ExpirationTimer(
self._current_hysteresis -
self._hysteresis_until.remaining())
def listen(self, timeout):
fulltime = ExpirationTimer(timeout)
while not fulltime.expired():
try:
response = self.my_instance.get(fulltime.remainingSeconds())
except Queue.Empty:
break
if not isInternalActorSystemMessage(response):
return response
return None
def retry(self, immediately=False):
if self._attempts > MAX_TRANSMIT_RETRIES:
return False
if self._quitTime.expired():
return False
self._attempts += 1
if immediately:
self._retryTime = ExpirationTimer(0)
else:
self._retryTime = ExpirationTimer(self._attempts * self.transmit_retry_period)
return True
class PauseWithBackoff(object):
def backoffPause(self, startPausing=False):
if startPausing:
self._lastPauseLength = backoffDelay(getattr(self, '_lastPauseLength', 0))
self._pauseUntil = ExpirationTimer(self._lastPauseLength)
return self._lastPauseLength
elif hasattr(self, '_pauseUntil'):
if not self._pauseUntil.expired():
return self._pauseUntil.remaining()
delattr(self, '_pauseUntil')
return timedelta(0)
def newPrimaryActor(self, actorClass, targetActorRequirements, globalName,
sourceHash=None):
self._numPrimaries = self._numPrimaries + 1
actorClassName = '%s.%s'%(actorClass.__module__, actorClass.__name__) \
if hasattr(actorClass, '__name__') else actorClass
response = NewActorResponse(self.transport, self.adminAddr)
self.transport.scheduleTransmit(
None,
TransmitIntent(self.adminAddr,
PendingActor(actorClassName,
None, self._numPrimaries,
targetActorRequirements,
globalName=globalName,
sourceHash=sourceHash),
onError=response.transmit_failed))
endwait = ExpirationTimer(MAX_CHILD_ACTOR_CREATE_DELAY)
# Do not use _run_transport: the tx_external transport
# context acquired above is unique to this thread and
# should not be synchronized/restricted by other threads.
self._run_transport(MAX_CHILD_ACTOR_CREATE_DELAY,
incomingHandler=response)
# Other items might abort the transport run... like transmit
# failures on a previous ask() that itself already timed out.
while response.pending and not endwait.expired():
self._run_transport(MAX_CHILD_ACTOR_CREATE_DELAY,
incomingHandler=response)
if response.failed:
if response.failure == PendingActorResponse.ERROR_Invalid_SourceHash:
raise InvalidActorSourceHash(sourceHash)
if response.failure == PendingActorResponse.ERROR_Invalid_ActorClass:
raise InvalidActorSpecification(actorClass,
response.failure_message)
if response.failure == PendingActorResponse.ERROR_Import:
info = response.failure_message
if info:
thesplog('Actor Create Failure, Import Error: %s', info)
raise ImportError(str(actorClass) + ': ' + info)
thesplog('Actor Create Failure, Import Error')
raise ImportError(actorClass)
if response.failure == PendingActorResponse.ERROR_No_Compatible_ActorSystem:
raise NoCompatibleSystemForActor(
actorClass, 'No compatible ActorSystem could be found')
raise ActorSystemFailure("Could not request new Actor from Admin (%s)"
% (response.failure))
if response.actor_address:
return response.actor_address
if response.actor_address is False:
raise NoCompatibleSystemForActor(
actorClass, 'No compatible ActorSystem could be found')
raise ActorSystemRequestTimeout(
'No response received to PendingActor request to Admin'
' at %s from %s'%(str(self.adminAddr),
str(self.transport.myAddress)))
def backoffPause(self, startPausing=False):
if startPausing:
self._lastPauseLength = backoffDelay(
getattr(self, '_lastPauseLength', 0))
self._pauseUntil = ExpirationTimer(self._lastPauseLength)
return self._lastPauseLength
elif hasattr(self, '_pauseUntil'):
if not self._pauseUntil.expired():
return self._pauseUntil.remaining()
delattr(self, '_pauseUntil')
return timedelta(0)
def __init__(self, myAddress, capabilities, sCBStats,
getConventionAddressFunc):
self._myAddress = myAddress
self._capabilities = capabilities
self._sCBStats = sCBStats
self._conventionMembers = AssocList() # key=Remote Admin Addr, value=ConventionMemberData
self._conventionNotificationHandlers = []
self._getConventionAddr = getConventionAddressFunc
self._conventionAddress = getConventionAddressFunc(capabilities)
self._conventionRegistration = ExpirationTimer(CONVENTION_REREGISTRATION_PERIOD)
self._has_been_activated = False
self._invited = False # entered convention as a result of an explicit invite
def testUnExpiredToUnExpiredComparison(self):
et1 = ExpirationTimer(timedelta(milliseconds=15))
et2 = ExpirationTimer(timedelta(milliseconds=10))
assert et1 != et2
assert et2 != et1
sleep(et2.remainingSeconds())
print(str(et1), str(et2))
assert et1 != et2
assert et2 != et1
sleep(et1.remainingSeconds())
assert et1 == et2
assert et2 == et1
def unloadActorSource(self, sourceHash):
loadLimit = ExpirationTimer(MAX_LOAD_SOURCE_DELAY)
txwatch = self._tx_to_admin(ValidateSource(sourceHash, None))
while not loadLimit.expired():
if not self._run_transport(loadLimit.remaining(), txonly=True):
return # all transmits completed
if txwatch.failed:
raise ActorSystemFailure(
'Error sending source unload to Admin: %s' %
str(txwatch.failure))
raise ActorSystemRequestTimeout('Unload source timeout: ' +
str(loadLimit))
def unloadActorSource(self, sourceHash):
loadLimit = ExpirationTimer(MAX_LOAD_SOURCE_DELAY)
txwatch = self._tx_to_admin(ValidateSource(sourceHash, None))
while not loadLimit.expired():
if not self._run_transport(loadLimit.remaining(), txonly=True):
return # all transmits completed
if txwatch.failed:
raise ActorSystemFailure(
'Error sending source unload to Admin: %s' %
str(txwatch.failure))
raise ActorSystemRequestTimeout('Unload source timeout: ' +
str(loadLimit))
def __init__(self,
actual_sender,
hysteresis_min_period=HYSTERESIS_MIN_PERIOD,
hysteresis_max_period=HYSTERESIS_MAX_PERIOD,
hysteresis_rate=HYSTERESIS_RATE):
self._sender = actual_sender
self._hysteresis_until = ExpirationTimer(timedelta(seconds=0))
self._hysteresis_queue = []
self._current_hysteresis = None # timedelta
self._hysteresis_min_period = hysteresis_min_period
self._hysteresis_max_period = hysteresis_max_period
self._hysteresis_rate = hysteresis_rate
def retry(self, immediately=False):
if self._attempts > MAX_TRANSMIT_RETRIES:
return False
if self._quitTime.view().expired():
return False
self._attempts += 1
if immediately:
self._retryTime = ExpirationTimer(0)
else:
self._retryTime = ExpirationTimer(self._attempts *
self.transmit_retry_period)
return True
def testNoneComparedToNonZero(self):
et1 = ExpirationTimer(None)
et2 = ExpirationTimer(timedelta(milliseconds=10))
# None == forever, so it is greater than anything, although equal to itself
assert et1 > et2
assert et2 < et1
assert et1 > et2
assert et2 < et1
sleep(et2.remainingSeconds())
assert et1 > et2
assert et2 < et1
assert et1 > et2
assert et2 < et1
def test_expiration_timer_None_period():
timer = ExpirationTimer(None)
with timer as t:
assert t.expired() == False
assert timer.view().expired() == False
time.sleep(0.2)
with timer as t:
assert t.expired() == False
assert t.remainingSeconds() == None
assert timer.view().expired() == False
assert timer.view().remainingSeconds() == None
def testNoneComparedToNonZero(self):
et1 = ExpirationTimer(None)
et2 = ExpirationTimer(timedelta(milliseconds=10))
# None == forever, so it is greater than anything, although equal to itself
assert et1 > et2
assert et2 < et1
assert et1 > et2
assert et2 < et1
sleep(et2.remainingSeconds())
assert et1 > et2
assert et2 < et1
assert et1 > et2
assert et2 < et1
def updateCapability(self, capabilityName, capabilityValue=None):
attemptLimit = ExpirationTimer(MAX_CAPABILITY_UPDATE_DELAY)
txwatch = self._tx_to_admin(
CapabilityUpdate(capabilityName, capabilityValue))
while not attemptLimit.expired():
if not self._run_transport(attemptLimit.remaining(), txonly=True):
return # all transmits completed
if txwatch.failed:
raise ActorSystemFailure(
'Error sending capability updates to Admin: %s' %
str(txwatch.failure))
raise ActorSystemRequestTimeout(
'Unable to confirm capability update in %s' %
str(MAX_CAPABILITY_UPDATE_DELAY))
def updateCapability(self, capabilityName, capabilityValue=None):
attemptLimit = ExpirationTimer(MAX_CAPABILITY_UPDATE_DELAY)
txwatch = self._tx_to_admin(CapabilityUpdate(capabilityName,
capabilityValue))
while not attemptLimit.expired():
if not self._run_transport(attemptLimit.remaining(), txonly=True):
return # all transmits completed
if txwatch.failed:
raise ActorSystemFailure(
'Error sending capability updates to Admin: %s' %
str(txwatch.failure))
raise ActorSystemRequestTimeout(
'Unable to confirm capability update in %s' %
str(MAX_CAPABILITY_UPDATE_DELAY))
def check_convention(self):
ct = currentTime()
rmsgs = []
if self._has_been_activated:
rmsgs = foldl(lambda x, y: x + y,
[self._check_preregistered_ping(ct, member)
for member in self._conventionMembers.values()],
self._convention_leader_checks(ct)
if self.isConventionLeader() or
not self.conventionLeaderAddr else
self._convention_member_checks(ct))
if self._conventionRegistration.view(ct).expired():
self._conventionRegistration = ExpirationTimer(CONVENTION_REREGISTRATION_PERIOD)
return rmsgs
def test_expiration_timer():
timer = ExpirationTimer(duration=1.0)
time.sleep(0.2)
with timer as t:
assert t.expired() == False
assert 0.7 <= t.remainingSeconds() <= 0.9
assert timer.view().expired() == False
assert 0.7 <= timer.view().remainingSeconds() <= 0.9
time.sleep(1.0)
with timer as t:
assert t.expired() == True
assert t.remainingSeconds() == 0.0
def run(self, incomingHandler, maximumDuration=None):
"""Core scheduling method; called by the current Actor process when
idle to await new messages (or to do background
processing).
"""
self._max_runtime = ExpirationTimer(maximumDuration)
# Always make at least one pass through to handle expired wakeups
# and queued events; otherwise a null/negative maximumDuration could
# block all processing.
firstTime = True
while firstTime or not self._max_runtime.expired():
firstTime = False
self._update_runtime()
rval = self._runWithExpiry(incomingHandler)
if rval is not None:
return rval
if not self._realizeWakeups():
# No wakeups were processed, and the inner run
# returned, so assume there's nothing to do and exit
return rval
while self._activeWakeups:
w = self._activeWakeups.pop()
if incomingHandler in (None, TransmitOnly):
return w
r = incomingHandler(w)
if not r:
return r
return None
def run(self):
# Main loop for convention management. Wraps the lower-level
# transport with a stop at the next needed convention
# registration period to re-register.
transport_continue = True
try:
while not getattr(self, 'shutdown_completed', False) and \
not isinstance(transport_continue, Thespian__Run_Terminated):
ct = currentTime()
delay = min(
self._cstate.convention_inattention_delay(ct),
ExpirationTimer(None).view(ct)
if self._hysteresisSender.delay.expired() else
self._hysteresisSender.delay)
# n.b. delay does not account for soon-to-expire
# pingValids, but since delay will not be longer than
# a CONVENTION_REREGISTRATION_PERIOD, the worst case
# is a doubling of a pingValid period (which should be fine).
transport_continue = self.transport.run(
self.handleIncoming, delay.remaining())
# Check Convention status based on the elapsed time
self._performIO(self._cstate.check_convention())
self._hysteresisSender.checkSends()
self._remove_expired_sources()
except Exception as ex:
import traceback
thesplog('ActorAdmin uncaught exception: %s',
traceback.format_exc(),
level=logging.ERROR,
exc_info=True)
thesplog('Admin time to die', level=logging.DEBUG)
def __init__(self, targetAddr, msg, onSuccess=None, onError=None, maxPeriod=None,
retryPeriod=TRANSMIT_RETRY_PERIOD):
super(TransmitIntent, self).__init__()
self._targetAddr = targetAddr
self._message = msg
self._callbackTo = ResultCallback(onSuccess, onError)
self._resultsts = None
self._quitTime = ExpirationTimer(maxPeriod or DEFAULT_MAX_TRANSMIT_PERIOD)
self._attempts = 0
self.transmit_retry_period = retryPeriod
def __init__(self, actual_sender,
hysteresis_min_period = HYSTERESIS_MIN_PERIOD,
hysteresis_max_period = HYSTERESIS_MAX_PERIOD,
hysteresis_rate = HYSTERESIS_RATE):
self._sender = actual_sender
self._hysteresis_until = ExpirationTimer(timedelta(seconds=0))
self._hysteresis_queue = []
self._current_hysteresis = None # timedelta
self._hysteresis_min_period = hysteresis_min_period
self._hysteresis_max_period = hysteresis_max_period
self._hysteresis_rate = hysteresis_rate
def _drain_tx_queue_if_needed(self, max_delay=None):
v, _ = self._complete_expired_intents()
if v >= MAX_QUEUED_TRANSMITS and self._aTB_rx_pause_enabled:
# Try to drain our local work before accepting more
# because it looks like we're getting really behind. This
# is dangerous though, because if other Actors are having
# the same issue this can create a deadlock.
thesplog('Entering tx-only mode to drain excessive queue'
' (%s > %s, drain-to %s)',
v, MAX_QUEUED_TRANSMITS,
QUEUE_TRANSMIT_UNBLOCK_THRESHOLD,
level=logging.WARNING)
finish_time = ExpirationTimer(max_delay if max_delay else None)
while v > QUEUE_TRANSMIT_UNBLOCK_THRESHOLD and not finish_time.expired():
if 0 == self.run(TransmitOnly, finish_time.remaining()):
thesplog('Exiting tx-only mode because no transport work available.')
break
v, _ = self._complete_expired_intents()
thesplog('Exited tx-only mode after draining excessive queue (%s)',
len(self._aTB_queuedPendingTransmits),
level=logging.WARNING)
def __init__(self, myAddress, capabilities, sCBStats,
getConventionAddressFunc):
self._myAddress = myAddress
self._capabilities = capabilities
self._sCBStats = sCBStats
self._conventionMembers = AssocList() # key=Remote Admin Addr, value=ConventionMemberData
self._conventionNotificationHandlers = []
self._getConventionAddr = getConventionAddressFunc
self._conventionAddress = getConventionAddressFunc(capabilities)
self._conventionRegistration = ExpirationTimer(CONVENTION_REREGISTRATION_PERIOD)
self._has_been_activated = False
self._invited = False # entered convention as a result of an explicit invite
def __init__(self, system, logDefs = None):
ensure_TZ_set()
# Expects self.transport has already been set by subclass __init__
super(systemBase, self).__init__(
self.transport.getAdminAddr(system.capabilities))
tryingTime = ExpirationTimer(MAX_SYSTEM_SHUTDOWN_DELAY + timedelta(seconds=1))
while not tryingTime.expired():
if not self.transport.probeAdmin(self.adminAddr):
self._startAdmin(self.adminAddr,
self.transport.myAddress,
system.capabilities,
logDefs)
if self._verifyAdminRunning(): return
import time
time.sleep(0.5) # Previous version may have been exiting
if not self._verifyAdminRunning():
raise InvalidActorAddress(self.adminAddr,
'not a valid or useable ActorSystem Admin')
def check_convention(self):
rmsgs = []
if self._has_been_activated:
rmsgs = foldl(lambda x, y: x + y,
[self._check_preregistered_ping(member)
for member in self._conventionMembers.values()],
self._convention_leader_checks()
if self.isConventionLeader() or
not self.conventionLeaderAddr else
self._convention_member_checks())
if self._conventionRegistration.expired():
self._conventionRegistration = ExpirationTimer(CONVENTION_REREGISTRATION_PERIOD)
return rmsgs
def loadActorSource(self, fname):
loadLimit = ExpirationTimer(MAX_LOAD_SOURCE_DELAY)
f = fname if hasattr(fname, 'read') else open(fname, 'rb')
try:
d = f.read()
import hashlib
hval = hashlib.md5(d).hexdigest()
txwatch = self._tx_to_admin(
ValidateSource(hval, d, getattr(f, 'name',
str(fname)
if hasattr(fname, 'read')
else fname)))
while not loadLimit.expired():
if not self._run_transport(loadLimit.remaining(), txonly=True):
# All transmits completed
return hval
if txwatch.failed:
raise ActorSystemFailure(
'Error sending source load to Admin: %s' %
str(txwatch.failure))
raise ActorSystemRequestTimeout('Load source timeout: ' +
str(loadLimit))
finally:
f.close()
def run(self, incomingHandler, maximumDuration=None):
"""Core scheduling method; called by the current Actor process when
idle to await new messages (or to do background
processing).
"""
self._max_runtime = ExpirationTimer(maximumDuration)
# Always make at least one pass through to handle expired wakeups
# and queued events; otherwise a null/negative maximumDuration could
# block all processing.
rval = self._run_subtransport(incomingHandler)
while rval in (True, None) and not self._max_runtime.expired():
rval = self._run_subtransport(incomingHandler)
return rval
def setup_convention(self, activation=False):
self._has_been_activated |= activation
rmsgs = []
# If not specified in capabilities, don't override any invites
# that may have been received.
self._conventionAddress = self._getConventionAddr(self.capabilities) or \
self._conventionAddress
leader_is_gone = (self._conventionMembers.find(self.conventionLeaderAddr) is None) \
if self.conventionLeaderAddr else True
if not self.isConventionLeader() and self.conventionLeaderAddr:
thesplog('Admin registering with Convention @ %s (%s)',
self.conventionLeaderAddr,
'first time' if leader_is_gone else 're-registering',
level=logging.INFO, primary=True)
rmsgs.append(
HysteresisSend(self.conventionLeaderAddr,
ConventionRegister(self.myAddress,
self.capabilities,
leader_is_gone),
onSuccess = self._setupConventionCBGood,
onError = self._setupConventionCBError))
rmsgs.append(LogAggregator(self.conventionLeaderAddr))
self._conventionRegistration = ExpirationTimer(CONVENTION_REREGISTRATION_PERIOD)
return rmsgs
class LocalConventionState(object):
def __init__(self, myAddress, capabilities, sCBStats,
getConventionAddressFunc):
self._myAddress = myAddress
self._capabilities = capabilities
self._sCBStats = sCBStats
self._conventionMembers = AssocList() # key=Remote Admin Addr, value=ConventionMemberData
self._conventionNotificationHandlers = []
self._getConventionAddr = getConventionAddressFunc
self._conventionAddress = getConventionAddressFunc(capabilities)
self._conventionRegistration = ExpirationTimer(CONVENTION_REREGISTRATION_PERIOD)
self._has_been_activated = False
self._invited = False # entered convention as a result of an explicit invite
@property
def myAddress(self):
return self._myAddress
@property
def capabilities(self):
return self._capabilities
def updateStatusResponse(self, resp):
resp.setConventionLeaderAddress(self.conventionLeaderAddr)
resp.setConventionRegisterTime(self._conventionRegistration)
for each in self._conventionMembers.values():
resp.addConventioneer(each.remoteAddress, each.registryValid)
resp.setNotifyHandlers(self._conventionNotificationHandlers)
def active_in_convention(self):
# If this is the convention leader, it is automatically
# active, otherwise this convention member should have a
# convention leader and that leader should have an active
# entry in the _conventionMembers table (indicating it has
# updated this system with its information)
return bool(self.conventionLeaderAddr and
self._conventionMembers.find(self.conventionLeaderAddr))
@property
def conventionLeaderAddr(self):
return self._conventionAddress
def isConventionLeader(self):
# Might also be the leader if self.conventionLeaderAddr is None
return self.conventionLeaderAddr == self.myAddress
def capabilities_have_changed(self, new_capabilities):
self._capabilities = new_capabilities
return self.setup_convention()
def setup_convention(self, activation=False):
self._has_been_activated |= activation
rmsgs = []
# If not specified in capabilities, don't override any invites
# that may have been received.
self._conventionAddress = self._getConventionAddr(self.capabilities) or \
self._conventionAddress
leader_is_gone = (self._conventionMembers.find(self.conventionLeaderAddr) is None) \
if self.conventionLeaderAddr else True
if not self.isConventionLeader() and self.conventionLeaderAddr:
thesplog('Admin registering with Convention @ %s (%s)',
self.conventionLeaderAddr,
'first time' if leader_is_gone else 're-registering',
level=logging.INFO, primary=True)
rmsgs.append(
HysteresisSend(self.conventionLeaderAddr,
ConventionRegister(self.myAddress,
self.capabilities,
leader_is_gone),
onSuccess = self._setupConventionCBGood,
onError = self._setupConventionCBError))
rmsgs.append(LogAggregator(self.conventionLeaderAddr))
self._conventionRegistration = ExpirationTimer(CONVENTION_REREGISTRATION_PERIOD)
return rmsgs
def _setupConventionCBGood(self, result, finishedIntent):
self._sCBStats.inc('Admin Convention Registered')
if hasattr(self, '_conventionLeaderMissCount'):
delattr(self, '_conventionLeaderMissCount')
def _setupConventionCBError(self, result, finishedIntent):
self._sCBStats.inc('Admin Convention Registration Failed')
if hasattr(self, '_conventionLeaderMissCount'):
self._conventionLeaderMissCount += 1
else:
self._conventionLeaderMissCount = 1
thesplog('Admin cannot register with convention @ %s (miss %d): %s',
finishedIntent.targetAddr,
self._conventionLeaderMissCount,
result, level=logging.WARNING, primary=True)
def got_convention_invite(self, sender):
self._conventionAddress = sender
self._invited = True
return self.setup_convention()
def got_convention_register(self, regmsg):
# Called when remote convention member has sent a ConventionRegister message
self._sCBStats.inc('Admin Handle Convention Registration')
if self._invited and not self.conventionLeaderAddr:
# Lost connection to an invitation-only convention.
# Cannot join again until another invitation is received.
return []
# Registrant may re-register if changing capabilities
rmsgs = []
registrant = regmsg.adminAddress
prereg = getattr(regmsg, 'preRegister', False) # getattr used; see definition
existing = self._conventionMembers.find(registrant)
thesplog('Got Convention %sregistration from %s (%s) (new? %s)',
'pre-' if prereg else '',
registrant,
'first time' if regmsg.firstTime else 're-registering',
not existing,
level=logging.INFO)
if registrant == self.myAddress:
# Either remote failed getting an external address and is
# using 127.0.0.1 or else this is a malicious attempt to
# make us talk to ourselves. Ignore it.
thesplog('Convention registration from %s is an invalid address; ignoring.',
registrant,
level=logging.WARNING)
return rmsgs
existingPreReg = (
# existing.preRegOnly
# or existing.preRegistered
existing.permanentEntry
) if existing else False
notify = (not existing or existing.preRegOnly) and not prereg
if regmsg.firstTime or not existing:
if existing:
existing = None
notify = not prereg
rmsgs.extend(self._remote_system_cleanup(registrant))
newmember = ConventionMemberData(registrant,
regmsg.capabilities,
prereg)
if prereg or existingPreReg:
newmember.preRegistered = PreRegistration()
self._conventionMembers.add(registrant, newmember)
else:
existing.refresh(regmsg.capabilities, prereg or existingPreReg)
if not prereg:
existing.preRegOnly = False
if not self.isConventionLeader():
self._conventionRegistration = ExpirationTimer(CONVENTION_REREGISTRATION_PERIOD)
# Convention Members normally periodically initiate a
# membership message, to which the leader confirms by
# responding; if this was a pre-registration, that identifies
# this system as the "leader" for that remote. Also, if the
# remote sent this because it was a pre-registration leader,
# it doesn't yet have all the member information so the member
# should respond.
#if self.isConventionLeader() or prereg or regmsg.firstTime:
if prereg:
rmsgs.append(HysteresisCancel(registrant))
rmsgs.append(TransmitIntent(registrant, ConventionInvite()))
elif (self.isConventionLeader() or prereg or regmsg.firstTime or \
(existing and existing.permanentEntry)):
# If we are the Convention Leader, this would be the point to
# inform all other registrants of the new registrant. At
# present, there is no reciprocity here, so just update the
# new registrant with the leader's info.
rmsgs.append(
TransmitIntent(registrant,
ConventionRegister(self.myAddress,
self.capabilities)))
if notify:
rmsgs.extend(self._notifications_of(
ActorSystemConventionUpdate(registrant,
regmsg.capabilities,
True)))
return rmsgs
def _notifications_of(self, msg):
return [TransmitIntent(H, msg) for H in self._conventionNotificationHandlers]
def add_notification_handler(self, addr):
if addr not in self._conventionNotificationHandlers:
self._conventionNotificationHandlers.append(addr)
# Now update the registrant on the current state of all convention members
return [TransmitIntent(addr,
ActorSystemConventionUpdate(M.remoteAddress,
M.remoteCapabilities,
True))
for M in self._conventionMembers.values()
if not M.preRegOnly]
return []
def remove_notification_handler(self, addr):
self._conventionNotificationHandlers = [
H for H in self._conventionNotificationHandlers
if H != addr]
def got_convention_deregister(self, deregmsg):
self._sCBStats.inc('Admin Handle Convention De-registration')
remoteAdmin = deregmsg.adminAddress
if remoteAdmin == self.myAddress:
# Either remote failed getting an external address and is
# using 127.0.0.1 or else this is a malicious attempt to
# make us talk to ourselves. Ignore it.
thesplog('Convention deregistration from %s is an invalid address; ignoring.',
remoteAdmin,
level=logging.WARNING)
rmsgs = []
if getattr(deregmsg, 'preRegistered', False): # see definition for getattr use
existing = self._conventionMembers.find(remoteAdmin)
if existing:
existing.preRegistered = None
rmsgs.append(TransmitIntent(remoteAdmin, ConventionDeRegister(self.myAddress)))
return rmsgs + self._remote_system_cleanup(remoteAdmin)
def got_system_shutdown(self):
return self.exit_convention()
def exit_convention(self):
self.invited = False
gen_ops = lambda addr: [HysteresisCancel(addr),
TransmitIntent(addr,
ConventionDeRegister(self.myAddress)),
]
terminate = lambda a: [ self._remote_system_cleanup(a), gen_ops(a) ][-1]
if self.conventionLeaderAddr and \
self.conventionLeaderAddr != self.myAddress:
thesplog('Admin de-registering with Convention @ %s',
str(self.conventionLeaderAddr),
level=logging.INFO, primary=True)
# Cache convention leader address because it might get reset by terminate()
claddr = self.conventionLeaderAddr
terminate(self.conventionLeaderAddr)
return gen_ops(claddr)
return join(fmap(terminate,
[M.remoteAddress
for M in self._conventionMembers.values()
if M.remoteAddress != self.myAddress]))
def check_convention(self):
rmsgs = []
if self._has_been_activated:
rmsgs = foldl(lambda x, y: x + y,
[self._check_preregistered_ping(member)
for member in self._conventionMembers.values()],
self._convention_leader_checks()
if self.isConventionLeader() or
not self.conventionLeaderAddr else
self._convention_member_checks())
if self._conventionRegistration.expired():
self._conventionRegistration = ExpirationTimer(CONVENTION_REREGISTRATION_PERIOD)
return rmsgs
def _convention_leader_checks(self):
return foldl(lambda x, y: x + y,
[self._missed_checkin_remote_cleanup(R)
for R in [ member
for member in self._conventionMembers.values()
if member.registryValid.expired() ]],
[])
def _missed_checkin_remote_cleanup(self, remote_member):
thesplog('%s missed %d checkins (%s); assuming it has died',
str(remote_member),
CONVENTION_REGISTRATION_MISS_MAX,
str(remote_member.registryValid),
level=logging.WARNING, primary=True)
return self._remote_system_cleanup(remote_member.remoteAddress)
def _convention_member_checks(self):
rmsgs = []
# Re-register with the Convention if it's time
if self.conventionLeaderAddr and self._conventionRegistration.expired():
if getattr(self, '_conventionLeaderMissCount', 0) >= \
CONVENTION_REGISTRATION_MISS_MAX:
thesplog('Admin convention registration lost @ %s (miss %d)',
self.conventionLeaderAddr,
self._conventionLeaderMissCount,
level=logging.WARNING, primary=True)
rmsgs.extend(self._remote_system_cleanup(self.conventionLeaderAddr))
self._conventionLeaderMissCount = 0
else:
rmsgs.extend(self.setup_convention())
return rmsgs
def _check_preregistered_ping(self, member):
if member.preRegistered and \
member.preRegistered.pingValid.expired() and \
not member.preRegistered.pingPending:
member.preRegistered.pingPending = True
# If remote misses a checkin, re-extend the
# invitation. This also helps re-initiate a socket
# connection if a TxOnly socket has been lost.
member.preRegistered.pingValid = ExpirationTimer(
convention_reinvite_adjustment(
CONVENTION_RESTART_PERIOD
if member.registryValid.expired()
else CONVENTION_REREGISTRATION_PERIOD))
return [HysteresisSend(member.remoteAddress,
ConventionInvite(),
onSuccess = self._preRegQueryNotPending,
onError = self._preRegQueryNotPending)]
return []
def _preRegQueryNotPending(self, result, finishedIntent):
remoteAddr = finishedIntent.targetAddr
member = self._conventionMembers.find(remoteAddr)
if member and member.preRegistered:
member.preRegistered.pingPending = False
def _remote_system_cleanup(self, registrant):
"""Called when a RemoteActorSystem has exited and all associated
Actors should be marked as exited and the ActorSystem
removed from Convention membership. This is also called on
a First Time connection from the remote to discard any
previous connection information.
"""
thesplog('Convention cleanup or deregistration for %s (known? %s)',
registrant,
bool(self._conventionMembers.find(registrant)),
level=logging.INFO)
rmsgs = [LostRemote(registrant)]
cmr = self._conventionMembers.find(registrant)
if not cmr or cmr.preRegOnly:
return []
# Send exited notification to conventionNotificationHandler (if any)
for each in self._conventionNotificationHandlers:
rmsgs.append(
TransmitIntent(each,
ActorSystemConventionUpdate(cmr.remoteAddress,
cmr.remoteCapabilities,
False))) # errors ignored
# If the remote ActorSystem shutdown gracefully (i.e. sent
# a Convention Deregistration) then it should not be
# necessary to shutdown remote Actors (or notify of their
# shutdown) because the remote ActorSystem should already
# have caused this to occur. However, it won't hurt, and
# it's necessary if the remote ActorSystem did not exit
# gracefully.
for lpa, raa in cmr.hasRemoteActors:
# ignore errors:
rmsgs.append(TransmitIntent(lpa, ChildActorExited(raa)))
# n.b. at present, this means that the parent might
# get duplicate notifications of ChildActorExited; it
# is expected that Actors can handle this.
# Remove remote system from conventionMembers
if not cmr.preRegistered:
if registrant == self.conventionLeaderAddr and self._invited:
self._conventionAddress = None
# Don't clear invited: once invited, that
# perpetually indicates this should be only a
# member and never a leader.
self._conventionMembers.rmv(registrant)
else:
# This conventionMember needs to stay because the
# current system needs to continue issuing
# registration pings. By setting the registryValid
# expiration to forever, this member won't re-time-out
# and will therefore be otherwise ignored... until it
# registers again at which point the membership will
# be updated with new settings.
cmr.registryValid = ExpirationTimer(None)
cmr.preRegOnly = True
return rmsgs + [HysteresisCancel(registrant)]
def sentByRemoteAdmin(self, envelope):
for each in self._conventionMembers.values():
if envelope.sender == each.remoteAddress:
return True
return False
def convention_inattention_delay(self):
return self._conventionRegistration or \
ExpirationTimer(CONVENTION_REREGISTRATION_PERIOD
if self.active_in_convention() or
self.isConventionLeader() else
CONVENTION_RESTART_PERIOD)
def forward_pending_to_remote_system(self, childClass, envelope, sourceHash, acceptsCaps):
alreadyTried = getattr(envelope.message, 'alreadyTried', [])
remoteCandidates = [
K
for K in self._conventionMembers.values()
if not K.registryValid.expired()
and K.remoteAddress != envelope.sender # source Admin
and K.remoteAddress not in alreadyTried
and acceptsCaps(K.remoteCapabilities)]
if not remoteCandidates:
if self.isConventionLeader() or not self.conventionLeaderAddr:
raise NoCompatibleSystemForActor(
childClass,
'No known ActorSystems can handle a %s for %s',
childClass, envelope.message.forActor)
# Let the Convention Leader try to find an appropriate ActorSystem
bestC = self.conventionLeaderAddr
else:
# distribute equally amongst candidates
C = [(K.remoteAddress, len(K.hasRemoteActors))
for K in remoteCandidates]
bestC = foldl(lambda best,possible:
best if best[1] <= possible[1] else possible,
C)[0]
thesplog('Requesting creation of %s%s on remote admin %s',
envelope.message.actorClassName,
' (%s)'%sourceHash if sourceHash else '',
bestC)
if bestC not in alreadyTried:
# Don't send request to this remote again, it has already
# been tried. This would also be indicated by that system
# performing the add of self.myAddress as below, but if
# there is disagreement between the local and remote
# addresses, this addition will prevent continual
# bounceback.
alreadyTried.append(bestC)
if self.myAddress not in alreadyTried:
# Don't send request back to this actor system: it cannot
# handle it
alreadyTried.append(self.myAddress)
envelope.message.alreadyTried = alreadyTried
return [TransmitIntent(bestC, envelope.message)]
def send_to_all_members(self, message, exception_list=None):
return [HysteresisSend(M.remoteAddress, message)
for M in self._conventionMembers.values()
if M.remoteAddress not in (exception_list or [])]
def got_convention_register(self, regmsg):
# Called when remote convention member has sent a ConventionRegister message
self._sCBStats.inc('Admin Handle Convention Registration')
if self._invited and not self.conventionLeaderAddr:
# Lost connection to an invitation-only convention.
# Cannot join again until another invitation is received.
return []
# Registrant may re-register if changing capabilities
rmsgs = []
registrant = regmsg.adminAddress
prereg = getattr(regmsg, 'preRegister', False) # getattr used; see definition
existing = self._conventionMembers.find(registrant)
thesplog('Got Convention %sregistration from %s (%s) (new? %s)',
'pre-' if prereg else '',
registrant,
'first time' if regmsg.firstTime else 're-registering',
not existing,
level=logging.INFO)
if registrant == self.myAddress:
# Either remote failed getting an external address and is
# using 127.0.0.1 or else this is a malicious attempt to
# make us talk to ourselves. Ignore it.
thesplog('Convention registration from %s is an invalid address; ignoring.',
registrant,
level=logging.WARNING)
return rmsgs
existingPreReg = (
# existing.preRegOnly
# or existing.preRegistered
existing.permanentEntry
) if existing else False
notify = (not existing or existing.preRegOnly) and not prereg
if regmsg.firstTime or not existing:
if existing:
existing = None
notify = not prereg
rmsgs.extend(self._remote_system_cleanup(registrant))
newmember = ConventionMemberData(registrant,
regmsg.capabilities,
prereg)
if prereg or existingPreReg:
newmember.preRegistered = PreRegistration()
self._conventionMembers.add(registrant, newmember)
else:
existing.refresh(regmsg.capabilities, prereg or existingPreReg)
if not prereg:
existing.preRegOnly = False
if not self.isConventionLeader():
self._conventionRegistration = ExpirationTimer(CONVENTION_REREGISTRATION_PERIOD)
# Convention Members normally periodically initiate a
# membership message, to which the leader confirms by
# responding; if this was a pre-registration, that identifies
# this system as the "leader" for that remote. Also, if the
# remote sent this because it was a pre-registration leader,
# it doesn't yet have all the member information so the member
# should respond.
#if self.isConventionLeader() or prereg or regmsg.firstTime:
if prereg:
rmsgs.append(HysteresisCancel(registrant))
rmsgs.append(TransmitIntent(registrant, ConventionInvite()))
elif (self.isConventionLeader() or prereg or regmsg.firstTime or \
(existing and existing.permanentEntry)):
# If we are the Convention Leader, this would be the point to
# inform all other registrants of the new registrant. At
# present, there is no reciprocity here, so just update the
# new registrant with the leader's info.
rmsgs.append(
TransmitIntent(registrant,
ConventionRegister(self.myAddress,
self.capabilities)))
if notify:
rmsgs.extend(self._notifications_of(
ActorSystemConventionUpdate(registrant,
regmsg.capabilities,
True)))
return rmsgs
class wakeupTransportBase(object):
"""The wakeupTransportBase is designed to be used as a mixin-base for
a Transport class and provides handling for the wakeupAfter()
functionality.
This base mixin provides the primary .run() entrypoint for the
transport and a .run_time ExpirationTime member that provides the
remaining time-to-run period.
The system can handle .wakeupAfter() requests by calling this
class's .addWakeup() method with the datetime.timedelta for the
wakeup to be scheduled.
The Transport should provide the following:
._runWithExpiry(incomingHandler)
Called by this class's .run() entrypoint to do the
actual transport-specific run routine. Should perform
that activity while the self.run_time ExpirationTimer is not
expired (self.run_time will be updated when new
wakeupAfter() events are scheduled).
"""
def __init__(self, *args, **kw):
super(wakeupTransportBase, self).__init__(*args, **kw)
# _pendingWakeups is a sorted list of ExpirationTimer objects,
# from the shortest to the longest.
self._pendingWakeups = []
self._activeWakeups = [] # expired wakeups to be delivered
def _updateStatusResponse(self, resp):
"""Called to update a Thespian_SystemStatus or Thespian_ActorStatus
with common information
"""
resp.addWakeups([(self.myAddress, T) for T in self._pendingWakeups])
for each in self._activeWakeups:
resp.addPendingMessage(self.myAddress, self.myAddress, str(each.message))
def _update_runtime(self):
self.run_time = (self._pendingWakeups + [self._max_runtime])[0]
def run(self, incomingHandler, maximumDuration=None):
"""Core scheduling method; called by the current Actor process when
idle to await new messages (or to do background
processing).
"""
self._max_runtime = ExpirationTimer(maximumDuration)
# Always make at least one pass through to handle expired wakeups
# and queued events; otherwise a null/negative maximumDuration could
# block all processing.
firstTime = True
while firstTime or not self._max_runtime.expired():
firstTime = False
self._update_runtime()
rval = self._runWithExpiry(incomingHandler)
if rval is not None:
return rval
if not self._realizeWakeups():
# No wakeups were processed, and the inner run
# returned, so assume there's nothing to do and exit
return rval
while self._activeWakeups:
w = self._activeWakeups.pop()
if incomingHandler in (None, TransmitOnly):
return w
r = incomingHandler(w)
if not r:
return r
return None
def addWakeup(self, timePeriod):
self._pendingWakeups.append(ExpirationTimer(timePeriod))
self._pendingWakeups.sort()
# The addWakeup method is called as a result of
# self.wakeupAfter, so ensure that the current run time is
# updated in case this new wakeup is the shortest.
self._update_runtime()
def _realizeWakeups(self):
"Find any expired wakeups and queue them to the send processing queue"
starting_len = len(self._activeWakeups)
while self._pendingWakeups and self._pendingWakeups[0].expired():
self._activeWakeups.append(
ReceiveEnvelope(self.myAddress,
WakeupMessage(self._pendingWakeups.pop(0).duration)))
return starting_len != len(self._activeWakeups)
class HysteresisDelaySender(object):
"""Implements hysteresis delay for sending messages. This is intended
to be used for messages exchanged between convention members to
ensure that a mis-behaved member doesn't have the ability to
inflict damage on the entire convention. The first time a
message is sent via this sender it is passed on through, but
that starts a blackout period that starts with the
CONVENTION_HYSTERESIS_MIN_PERIOD. Each additional send attempt
during that blackout period will cause the blackout period to
be extended by the CONVENTION_HYSTERESIS_RATE, up to the
CONVENTION_HYSTERESIS_MAX_PERIOD. Once the blackout period
ends, the queued sends will be sent, but only the last
attempted message of each type for the specified remote target.
At that point, the hysteresis delay will be reduced by the
CONVENTION_HYSTERESIS_RATE; further send attempts will affect
the hysteresis blackout period as described as above but lack
of sending attempts will continue to reduce the hysteresis back
to a zero-delay setting.
Note: delays are updated in a target-independent manner; the
target is only considered when eliminating duplicates.
Note: maxDelay on TransmitIntents is ignored by hysteresis
delays. It is assumed that a transmit intent's maxDelay
is greater than the maximum hysteresis period and/or that
the hysteresis delay is more important than the transmit
intent timeout.
"""
def __init__(self, actual_sender,
hysteresis_min_period = HYSTERESIS_MIN_PERIOD,
hysteresis_max_period = HYSTERESIS_MAX_PERIOD,
hysteresis_rate = HYSTERESIS_RATE):
self._sender = actual_sender
self._hysteresis_until = ExpirationTimer(timedelta(seconds=0))
self._hysteresis_queue = []
self._current_hysteresis = None # timedelta
self._hysteresis_min_period = hysteresis_min_period
self._hysteresis_max_period = hysteresis_max_period
self._hysteresis_rate = hysteresis_rate
@property
def delay(self):
return self._hysteresis_until
def _has_hysteresis(self):
return (self._current_hysteresis is not None and
self._current_hysteresis >= self._hysteresis_min_period)
def _increase_hysteresis(self):
if self._has_hysteresis():
try:
self._current_hysteresis = min(
(self._current_hysteresis * self._hysteresis_rate),
self._hysteresis_max_period)
except TypeError:
# See note below for _decrease_hysteresis
self._current_hysteresis = min(
timedelta(
seconds=(self._current_hysteresis.seconds *
self._hysteresis_rate)),
self._hysteresis_max_period)
else:
self._current_hysteresis = self._hysteresis_min_period
def _decrease_hysteresis(self):
try:
self._current_hysteresis = (
(self._current_hysteresis / self._hysteresis_rate)
if self._has_hysteresis() else None)
except TypeError:
# Python 2.x cannot multiply or divide a timedelta by a
# fractional amount. There is also not a total_seconds
# retrieval from a timedelta, but it should be safe to
# assume that the hysteresis value is not greater than 1
# day.
self._current_hysteresis = timedelta(
seconds=(self._current_hysteresis.seconds /
self._hysteresis_rate)) \
if self._has_hysteresis() else None
def _update_remaining_hysteresis_period(self, reset=False):
if not self._current_hysteresis:
self._hysteresis_until = ExpirationTimer(timedelta(seconds=0))
else:
if reset or not self._hysteresis_until:
self._hysteresis_until = ExpirationTimer(self._current_hysteresis)
else:
self._hysteresis_until = ExpirationTimer(
self._current_hysteresis -
self._hysteresis_until.remaining())
def checkSends(self):
if self.delay.expired():
self._decrease_hysteresis()
self._update_remaining_hysteresis_period(reset=True)
for intent in self._keepIf(lambda M: False):
self._sender(intent)
@staticmethod
def safe_cmp(val1, val2):
try:
return val1 == val2
except Exception:
return False
def sendWithHysteresis(self, intent):
if self._hysteresis_until.expired():
self._current_hysteresis = self._hysteresis_min_period
self._sender(intent)
else:
dups = self._keepIf(lambda M:
(M.targetAddr != intent.targetAddr or
not HysteresisDelaySender
.safe_cmp(M.message, intent.message)))
# The dups are duplicate sends to the new intent's target;
# complete them when the actual message is finally sent
# with the same result
if dups:
intent.addCallback(self._dupSentGood(dups),
self._dupSentFail(dups))
self._hysteresis_queue.append(intent)
self._increase_hysteresis()
self._update_remaining_hysteresis_period()
def cancelSends(self, remoteAddr):
for each in self._keepIf(lambda M: M.targetAddr != remoteAddr):
each.tx_done(SendStatus.Failed)
def _keepIf(self, keepFunc):
requeues, removes = partition(keepFunc, self._hysteresis_queue)
self._hysteresis_queue = requeues
return removes
@staticmethod
def _dupSentGood(dups):
def _finishDups(result, finishedIntent):
for each in dups:
each.tx_done(result)
return _finishDups
@staticmethod
def _dupSentFail(dups):
def _finishDups(result, finishedIntent):
for each in dups:
each.tx_done(result)
return _finishDups
