def run(self):
# Main loop for convention management. Wraps the lower-level
# transport with a stop at the next needed convention
# registration period to re-register.
try:
while not self.isShuttingDown():
delay = min(self._conventionRegistration or \
ExpiryTime(CONVENTION_RESTART_PERIOD
if self._conventionLost and not self.isConventionLeader() else
CONVENTION_REREGISTRATION_PERIOD),
ExpiryTime(None) if self._hysteresisSender.delay.expired() else
self._hysteresisSender.delay
)
# n.b. delay does not account for soon-to-expire
# pingValids, but since delay will not be longer than
# a CONVENTION_REREGISTRATION_PERIOD, the worst case
# is a doubling of a pingValid period (which should be fine).
self.transport.run(self.handleIncoming, delay.remaining())
self._checkConvention()
self._hysteresisSender.checkSends()
except Exception as ex:
import traceback
thesplog('ActorAdmin uncaught exception: %s',
traceback.format_exc(),
level=logging.ERROR,
exc_info=True)
thesplog('Admin time to die', level=logging.DEBUG)
def _checkConvention(self):
if self.isConventionLeader():
missing = []
for each in self._conventionMembers.values():
if each.registryValid.expired():
missing.append(each)
for each in missing:
thesplog('%s missed %d checkins (%s); assuming it has died',
str(each),
CONVENTION_REGISTRATION_MISS_MAX,
str(each.registryValid),
level=logging.WARNING,
primary=True)
self._remoteSystemCleanup(each.remoteAddress)
self._conventionRegistration = ExpiryTime(
CONVENTION_REREGISTRATION_PERIOD)
else:
# Re-register with the Convention if it's time
if self._conventionAddress and self._conventionRegistration.expired(
):
self.setupConvention()
for member in self._conventionMembers.values():
if member.preRegistered and \
member.preRegistered.pingValid.expired() and \
not member.preRegistered.pingPending:
member.preRegistered.pingPending = True
member.preRegistered.pingValid = ExpiryTime(
CONVENTION_RESTART_PERIOD if member.registryValid.expired(
) else CONVENTION_REREGISTRATION_PERIOD)
self._hysteresisSender.sendWithHysteresis(
TransmitIntent(member.remoteAddress,
ConventionInvite(),
onSuccess=self._preRegQueryNotPending,
onError=self._preRegQueryNotPending))
def run(self, incomingHandler, maximumDuration=None):
"""Core scheduling method; called by the current Actor process when
idle to await new messages (or to do background
processing).
"""
self._max_runtime = ExpiryTime(maximumDuration)
while not self._max_runtime.expired():
now = datetime.now()
self.run_time = min(
[ExpiryTime(P - now)
for P in self._pendingWakeups] + [self._max_runtime])
rval = self._runWithExpiry(incomingHandler)
if rval is not None:
return rval
if not self._realizeWakeups():
# No wakeups were processed, and the inner run
# returned, so assume there's nothing to do and exit
return rval
while self._activeWakeups:
w = self._activeWakeups.pop()
if incomingHandler is None:
return w
if not incomingHandler(w):
return None
return None
def run(self, incomingHandler, maximumDuration=None):
"""Core scheduling method; called by the current Actor process when
idle to await new messages (or to do background
processing).
"""
self._max_runtime = ExpiryTime(maximumDuration)
# Always make at least one pass through to handle expired wakeups
# and queued events; otherwise a null/negative maximumDuration could
# block all processing.
firstTime = True
while firstTime or not self._max_runtime.expired():
firstTime = False
now = datetime.now()
self.run_time = min([ExpiryTime(P - now) for P in self._pendingWakeups] +
[self._max_runtime])
rval = self._runWithExpiry(incomingHandler)
if rval is not None:
return rval
if not self._realizeWakeups():
# No wakeups were processed, and the inner run
# returned, so assume there's nothing to do and exit
return rval
while self._activeWakeups:
w = self._activeWakeups.pop()
if incomingHandler in (None, TransmitOnly):
return w
if not incomingHandler(w):
return None
return None
def testNoneComparedToZero(self):
et1 = ExpiryTime(None)
et2 = ExpiryTime(timedelta(days=0))
# None == forever, so it is greater than anything, although equal to itself
self.assertGreater(et1, et2)
self.assertLess(et2, et1)
self.assertTrue(et1 >= et2)
self.assertTrue(et2 <= et1)
def testNoneToUnExpiredComparison(self):
et1 = ExpiryTime(None)
et2 = ExpiryTime(timedelta(milliseconds=10))
self.assertNotEqual(et1, et2)
self.assertNotEqual(et2, et1)
sleep(et2.remainingSeconds())
self.assertNotEqual(et1, et2)
self.assertNotEqual(et2, et1)
def testNoneToUnExpiredComparison(self):
et1 = ExpiryTime(None)
et2 = ExpiryTime(timedelta(milliseconds=10))
assert et1 != et2
assert et2 != et1
sleep(et2.remainingSeconds())
assert et1 != et2
assert et2 != et1
def testNoneComparedToZero(self):
et1 = ExpiryTime(None)
et2 = ExpiryTime(timedelta(days=0))
# None == forever, so it is greater than anything, although equal to itself
assert et1 > et2
assert et2 < et1
assert et1 >= et2
assert et2 <= et1
def _update_remaining_hysteresis_period(self, reset=False):
if not self._current_hysteresis:
self._hysteresis_until = ExpiryTime(timedelta(seconds=0))
else:
if reset or not self._hysteresis_until:
self._hysteresis_until = ExpiryTime(self._current_hysteresis)
else:
self._hysteresis_until = ExpiryTime(
self._current_hysteresis -
self._hysteresis_until.remaining())
def testUnExpiredToUnExpiredComparison(self):
et1 = ExpiryTime(timedelta(milliseconds=15))
et2 = ExpiryTime(timedelta(milliseconds=10))
assert et1 != et2
assert et2 != et1
sleep(et2.remainingSeconds())
print(str(et1), str(et2))
assert et1 != et2
assert et2 != et1
sleep(et1.remainingSeconds())
assert et1 == et2
assert et2 == et1
def testNoneComparedToNonZero(self):
et1 = ExpiryTime(None)
et2 = ExpiryTime(timedelta(milliseconds=10))
# None == forever, so it is greater than anything, although equal to itself
self.assertGreater(et1, et2)
self.assertLess(et2, et1)
self.assertTrue(et1 > et2)
self.assertTrue(et2 < et1)
sleep(et2.remainingSeconds())
self.assertGreater(et1, et2)
self.assertLess(et2, et1)
self.assertTrue(et1 > et2)
self.assertTrue(et2 < et1)
def testNoneComparedToNonZero(self):
et1 = ExpiryTime(None)
et2 = ExpiryTime(timedelta(milliseconds=10))
# None == forever, so it is greater than anything, although equal to itself
assert et1 > et2
assert et2 < et1
assert et1 > et2
assert et2 < et1
sleep(et2.remainingSeconds())
assert et1 > et2
assert et2 < et1
assert et1 > et2
assert et2 < et1
def testNonZeroIsFalse(self):
et = ExpiryTime(timedelta(milliseconds=10))
assert not et
assert not bool(et)
sleep(et.remainingSeconds())
assert et
assert bool(et)
def shutdown(self):
thesplog('ActorSystem shutdown requested.', level=logging.INFO)
time_to_quit = ExpiryTime(MAX_SYSTEM_SHUTDOWN_DELAY)
self.transport.scheduleTransmit(
None,
TransmitIntent(self.adminAddr,
SystemShutdown(),
onError=self._shutdownSendFailed))
while not time_to_quit.expired():
response = self.transport.run(None, time_to_quit.remaining())
if getattr(self, '_TASF', False):
thesplog(
'Could not send shutdown request to Admin'
'; aborting but not necessarily stopped',
level=logging.WARNING)
return
if response:
if isinstance(response.message, SystemShutdownCompleted):
break
else:
thesplog('Expected shutdown completed message, got: %s',
response.message,
level=logging.WARNING)
else:
thesplog(
'No response to Admin shutdown request; Actor system not completely shutdown',
level=logging.ERROR)
thesplog('ActorSystem shutdown complete.')
def run(self):
# Main loop for convention management. Wraps the lower-level
# transport with a stop at the next needed convention
# registration period to re-register.
try:
while not getattr(self, 'shutdown_completed', False):
delay = min(
self._cstate.convention_inattention_delay(),
ExpiryTime(None) if self._hysteresisSender.delay.expired()
else self._hysteresisSender.delay)
# n.b. delay does not account for soon-to-expire
# pingValids, but since delay will not be longer than
# a CONVENTION_REREGISTRATION_PERIOD, the worst case
# is a doubling of a pingValid period (which should be fine).
r = self.transport.run(self.handleIncoming, delay.remaining())
# Check Convention status based on the elapsed time
self._performIO(self._cstate.check_convention())
self._hysteresisSender.checkSends()
except Exception as ex:
import traceback
thesplog('ActorAdmin uncaught exception: %s',
traceback.format_exc(),
level=logging.ERROR,
exc_info=True)
thesplog('Admin time to die', level=logging.DEBUG)
def sendWithHysteresis(self, intent):
if self._hysteresis_until.expired():
self._current_hysteresis = self._hysteresis_min_period
self._sender(intent)
else:
dups = self._keepIf(
lambda M: (M.targetAddr != intent.targetAddr or type(M.message)
!= type(intent.message)))
# The dups are duplicate sends to the new intent's target; complete them when
# the actual message is finally sent with the same result
if dups:
intent.addCallback(self._dupSentGood(dups),
self._dupSentFail(dups))
self._hysteresis_queue.append(intent)
self._current_hysteresis = min(
(self._hysteresis_min_period if
(self._current_hysteresis is None
or self._current_hysteresis < self._hysteresis_min_period)
else self._current_hysteresis * self._hysteresis_rate),
self._hysteresis_max_period)
self._hysteresis_until = ExpiryTime(
timedelta(seconds=0) if not self._current_hysteresis else (
self._current_hysteresis -
(timedelta(seconds=0) if not self._hysteresis_until else self.
_hysteresis_until.remaining())))
def loadActorSource(self, fname):
self._LOADFAILED = None
loadLimit = ExpiryTime(MAX_LOAD_SOURCE_DELAY)
f = fname if hasattr(fname, 'read') else open(fname, 'rb')
try:
d = f.read()
import hashlib
hval = hashlib.md5(d).hexdigest()
self.transport.scheduleTransmit(
None,
TransmitIntent(
self.adminAddr,
ValidateSource(
hval, d,
getattr(
f, 'name',
str(fname) if hasattr(fname, 'read') else fname)),
onError=self._loadReqFailed))
while not loadLimit.expired():
if not self.transport.run(TransmitOnly, loadLimit.remaining()):
break # all transmits completed
if self._LOADFAILED or loadLimit.expired():
raise ActorSystemFailure('Load source failed due to ' + (
'failure response (%s)' %
self._LOADFAILED if self._LOADFAILED else 'timeout (%s)' %
str(loadLimit)))
return hval
finally:
f.close()
def setupConvention(self):
if self.isShuttingDown(): return
if not self._conventionAddress:
gCA = getattr(self.transport, 'getConventionAddress',
lambda c: None)
self._conventionAddress = gCA(self.capabilities)
if self._conventionAddress == self.myAddress:
self._conventionAddress = None
if self._conventionAddress:
thesplog('Admin registering with Convention @ %s (%s)',
self._conventionAddress,
'first time' if getattr(self, '_conventionLeaderIsGone',
True) else 're-registering',
level=logging.INFO,
primary=True)
self._hysteresisSender.sendWithHysteresis(
TransmitIntent(self._conventionAddress,
ConventionRegister(
self.myAddress, self.capabilities,
getattr(self, '_conventionLeaderIsGone',
True)),
onSuccess=self._setupConventionCBGood,
onError=self._setupConventionCBError))
self._conventionRegistration = ExpiryTime(
CONVENTION_REREGISTRATION_PERIOD)
def _remoteSystemCleanup(self, registrant):
"""Called when a RemoteActorSystem has exited and all associated
Actors should be marked as exited and the ActorSystem
removed from Convention membership.
"""
thesplog('Convention cleanup or deregistration for %s (new? %s)',
registrant,
registrant.actorAddressString not in self._conventionMembers,
level=logging.INFO)
if registrant.actorAddressString in self._conventionMembers:
cmr = self._conventionMembers[registrant.actorAddressString]
# Send exited notification to conventionNotificationHandler (if any)
if self.isConventionLeader():
for each in self._conventionNotificationHandlers:
self._send_intent(
TransmitIntent(
each,
ActorSystemConventionUpdate(
cmr.remoteAddress, cmr.remoteCapabilities,
False))) # errors ignored
# If the remote ActorSystem shutdown gracefully (i.e. sent
# a Convention Deregistration) then it should not be
# necessary to shutdown remote Actors (or notify of their
# shutdown) because the remote ActorSystem should already
# have caused this to occur. However, it won't hurt, and
# it's necessary if the remote ActorSystem did not exit
# gracefully.
for lpa, raa in cmr.hasRemoteActors:
# ignore errors:
self._send_intent(TransmitIntent(lpa, ChildActorExited(raa)))
# n.b. at present, this means that the parent might
# get duplicate notifications of ChildActorExited; it
# is expected that Actors can handle this.
# Remove remote system from conventionMembers
if not cmr.preRegistered:
del self._conventionMembers[registrant.actorAddressString]
else:
# This conventionMember needs to stay because the
# current system needs to continue issuing
# registration pings. By setting the registryValid
# expiration to forever, this member won't re-time-out
# and will therefore be otherwise ignored... until it
# registers again at which point the membership will
# be updated with new settings.
cmr.registryValid = ExpiryTime(None)
if registrant == self._conventionAddress:
# Convention Leader has exited. Do NOT set
# conventionAddress to None. It might speed up shutdown
# of this ActorSystem because it won't try to de-register
# from the convention leader, but if the convention leader
# reappears there will be nothing to get this ActorSystem
# re-registered with the convention.
self._conventionLeaderIsGone = True
self._hysteresisSender.cancelSends(registrant)
def addWakeup(self, timePeriod):
now = datetime.now()
wakeupTime = now + timePeriod
self._pendingWakeups.setdefault(wakeupTime, []) \
.append(ReceiveEnvelope(self.myAddress, WakeupMessage(timePeriod)))
self.run_time = min([ExpiryTime(P - now) for P in self._pendingWakeups] +
[self._max_runtime])
def testNonZeroIsFalse(self):
et = ExpiryTime(timedelta(milliseconds=10))
self.assertFalse(et)
self.assertFalse(bool(et))
sleep(et.remainingSeconds())
self.assertTrue(et)
self.assertTrue(bool(et))
def __init__(self, address, capabilities):
self.remoteAddress = address
self.remoteCapabilities = capabilities
self.registryValid = ExpiryTime(CONVENTION_REREGISTRATION_PERIOD *
CONVENTION_REGISTRATION_MISS_MAX)
self.hasRemoteActors = [
] # (localParent, remoteActor) addresses created remotely
self.lastMessaged = None # datetime of access; use with CONVENTION_HYSTERESIS_PERIOD
def __init__(self, *args, **kw):
super(ConventioneerAdmin, self).__init__(*args, **kw)
self._conventionMembers = {
} # key=Remote Admin Addr, value=ConventionMemberData
self._conventionRegistration = ExpiryTime(timedelta(seconds=0))
self._conventionNotificationHandlers = set()
self._conventionAddress = None # Not a member; still could be a leader
self._pendingSources = {
} # key = sourceHash, value is array of PendingActor requests
self._hysteresisSender = HysteresisDelaySender(self._send_intent)
def _reset_valid_timer(self):
# registryValid is a timer that is usually set to a multiple
# of the convention re-registration period. Each successful
# convention re-registration resets the timer to the maximum
# value (actually, it replaces this structure with a newly
# generated structure). If the timer expires, the remote is
# declared as dead and the registration is removed (or
# quiesced if it is a pre-registration).
self.registryValid = ExpiryTime(CONVENTION_REREGISTRATION_PERIOD *
CONVENTION_REGISTRATION_MISS_MAX)
def __init__(self, actual_sender,
hysteresis_min_period = HYSTERESIS_MIN_PERIOD,
hysteresis_max_period = HYSTERESIS_MAX_PERIOD,
hysteresis_rate = HYSTERESIS_RATE):
self._sender = actual_sender
self._hysteresis_until = ExpiryTime(timedelta(seconds=0))
self._hysteresis_queue = []
self._current_hysteresis = None # timedelta
self._hysteresis_min_period = hysteresis_min_period
self._hysteresis_max_period = hysteresis_max_period
self._hysteresis_rate = hysteresis_rate
def run(self):
try:
while not self.isShuttingDown():
delay = min(self._conventionRegistration or \
ExpiryTime(CONVENTION_RESTART_PERIOD
if self._conventionLost and not self.isConventionLeader() else
CONVENTION_REREGISTRATION_PERIOD),
ExpiryTime(None) if self._hysteresisSender.delay.expired() else
self._hysteresisSender.delay
)
self.transport.run(self.handleIncoming, delay.remaining())
self._checkConvention()
self._hysteresisSender.checkSends()
except Exception as ex:
import traceback
thesplog('ActorAdmin uncaught exception: %s',
traceback.format_exc(),
level=logging.ERROR,
exc_info=True)
thesplog('Admin time to die', level=logging.DEBUG)
def _remote_system_cleanup(self, registrant):
"""Called when a RemoteActorSystem has exited and all associated
Actors should be marked as exited and the ActorSystem
removed from Convention membership. This is also called on
a First Time connection from the remote to discard any
previous connection information.
"""
thesplog('Convention cleanup or deregistration for %s (known? %s)',
registrant,
bool(self._conventionMembers.find(registrant)),
level=logging.INFO)
rmsgs = [LostRemote(registrant)]
cmr = self._conventionMembers.find(registrant)
if cmr:
# Send exited notification to conventionNotificationHandler (if any)
for each in self._conventionNotificationHandlers:
rmsgs.append(
TransmitIntent(
each,
ActorSystemConventionUpdate(cmr.remoteAddress,
cmr.remoteCapabilities,
False))) # errors ignored
# If the remote ActorSystem shutdown gracefully (i.e. sent
# a Convention Deregistration) then it should not be
# necessary to shutdown remote Actors (or notify of their
# shutdown) because the remote ActorSystem should already
# have caused this to occur. However, it won't hurt, and
# it's necessary if the remote ActorSystem did not exit
# gracefully.
for lpa, raa in cmr.hasRemoteActors:
# ignore errors:
rmsgs.append(TransmitIntent(lpa, ChildActorExited(raa)))
# n.b. at present, this means that the parent might
# get duplicate notifications of ChildActorExited; it
# is expected that Actors can handle this.
# Remove remote system from conventionMembers
if not cmr.preRegistered:
self._conventionMembers.rmv(registrant)
else:
# This conventionMember needs to stay because the
# current system needs to continue issuing
# registration pings. By setting the registryValid
# expiration to forever, this member won't re-time-out
# and will therefore be otherwise ignored... until it
# registers again at which point the membership will
# be updated with new settings.
cmr.registryValid = ExpiryTime(None)
return rmsgs + [HysteresisCancel(registrant)]
def updateCapability(self, capabilityName, capabilityValue=None):
self._updCAPFAILED = False
attemptLimit = ExpiryTime(MAX_CAPABILITY_UPDATE_DELAY)
self.transport.scheduleTransmit(
None,
TransmitIntent(self.adminAddr,
CapabilityUpdate(capabilityName, capabilityValue),
onError = self._updateCapsFailed))
while not attemptLimit.expired():
if not self.transport.run(TransmitOnly, attemptLimit.remaining()):
break # all transmits completed
if self._updCAPFAILED or attemptLimit.expired():
raise ActorSystemFailure("Could not update Actor System Admin capabilities.")
def __init__(self, myAddress, capabilities, sCBStats,
getConventionAddressFunc):
self._myAddress = myAddress
self._capabilities = capabilities
self._sCBStats = sCBStats
self._conventionMembers = AssocList(
) # key=Remote Admin Addr, value=ConventionMemberData
self._conventionNotificationHandlers = []
self._getConventionAddr = getConventionAddressFunc
self._conventionAddress = getConventionAddressFunc(capabilities)
self._conventionRegistration = ExpiryTime(
CONVENTION_REREGISTRATION_PERIOD)
self._has_been_activated = False
def check_convention(self):
rmsgs = []
if not self._has_been_activated:
return rmsgs
if self.isConventionLeader() or not self.conventionLeaderAddr:
missing = []
for each in self._conventionMembers.values():
if each.registryValid.expired():
missing.append(each)
for each in missing:
thesplog('%s missed %d checkins (%s); assuming it has died',
str(each),
CONVENTION_REGISTRATION_MISS_MAX,
str(each.registryValid),
level=logging.WARNING,
primary=True)
rmsgs.extend(self._remote_system_cleanup(each.remoteAddress))
self._conventionRegistration = ExpiryTime(
CONVENTION_REREGISTRATION_PERIOD)
else:
# Re-register with the Convention if it's time
if self.conventionLeaderAddr and self._conventionRegistration.expired(
):
rmsgs.extend(self.setup_convention())
for member in self._conventionMembers.values():
if member.preRegistered and \
member.preRegistered.pingValid.expired() and \
not member.preRegistered.pingPending:
member.preRegistered.pingPending = True
member.preRegistered.pingValid = ExpiryTime(
CONVENTION_RESTART_PERIOD if member.registryValid.expired(
) else CONVENTION_REREGISTRATION_PERIOD)
rmsgs.append(
HysteresisSend(member.remoteAddress,
ConventionInvite(),
onSuccess=self._preRegQueryNotPending,
onError=self._preRegQueryNotPending))
return rmsgs
