def run(self): # Main loop for convention management. Wraps the lower-level # transport with a stop at the next needed convention # registration period to re-register. try: while not self.isShuttingDown(): delay = min(self._conventionRegistration or \ ExpiryTime(CONVENTION_RESTART_PERIOD if self._conventionLost and not self.isConventionLeader() else CONVENTION_REREGISTRATION_PERIOD), ExpiryTime(None) if self._hysteresisSender.delay.expired() else self._hysteresisSender.delay ) # n.b. delay does not account for soon-to-expire # pingValids, but since delay will not be longer than # a CONVENTION_REREGISTRATION_PERIOD, the worst case # is a doubling of a pingValid period (which should be fine). self.transport.run(self.handleIncoming, delay.remaining()) self._checkConvention() self._hysteresisSender.checkSends() except Exception as ex: import traceback thesplog('ActorAdmin uncaught exception: %s', traceback.format_exc(), level=logging.ERROR, exc_info=True) thesplog('Admin time to die', level=logging.DEBUG)
def _checkConvention(self): if self.isConventionLeader(): missing = [] for each in self._conventionMembers.values(): if each.registryValid.expired(): missing.append(each) for each in missing: thesplog('%s missed %d checkins (%s); assuming it has died', str(each), CONVENTION_REGISTRATION_MISS_MAX, str(each.registryValid), level=logging.WARNING, primary=True) self._remoteSystemCleanup(each.remoteAddress) self._conventionRegistration = ExpiryTime( CONVENTION_REREGISTRATION_PERIOD) else: # Re-register with the Convention if it's time if self._conventionAddress and self._conventionRegistration.expired( ): self.setupConvention() for member in self._conventionMembers.values(): if member.preRegistered and \ member.preRegistered.pingValid.expired() and \ not member.preRegistered.pingPending: member.preRegistered.pingPending = True member.preRegistered.pingValid = ExpiryTime( CONVENTION_RESTART_PERIOD if member.registryValid.expired( ) else CONVENTION_REREGISTRATION_PERIOD) self._hysteresisSender.sendWithHysteresis( TransmitIntent(member.remoteAddress, ConventionInvite(), onSuccess=self._preRegQueryNotPending, onError=self._preRegQueryNotPending))
def run(self, incomingHandler, maximumDuration=None): """Core scheduling method; called by the current Actor process when idle to await new messages (or to do background processing). """ self._max_runtime = ExpiryTime(maximumDuration) while not self._max_runtime.expired(): now = datetime.now() self.run_time = min( [ExpiryTime(P - now) for P in self._pendingWakeups] + [self._max_runtime]) rval = self._runWithExpiry(incomingHandler) if rval is not None: return rval if not self._realizeWakeups(): # No wakeups were processed, and the inner run # returned, so assume there's nothing to do and exit return rval while self._activeWakeups: w = self._activeWakeups.pop() if incomingHandler is None: return w if not incomingHandler(w): return None return None
def run(self, incomingHandler, maximumDuration=None): """Core scheduling method; called by the current Actor process when idle to await new messages (or to do background processing). """ self._max_runtime = ExpiryTime(maximumDuration) # Always make at least one pass through to handle expired wakeups # and queued events; otherwise a null/negative maximumDuration could # block all processing. firstTime = True while firstTime or not self._max_runtime.expired(): firstTime = False now = datetime.now() self.run_time = min([ExpiryTime(P - now) for P in self._pendingWakeups] + [self._max_runtime]) rval = self._runWithExpiry(incomingHandler) if rval is not None: return rval if not self._realizeWakeups(): # No wakeups were processed, and the inner run # returned, so assume there's nothing to do and exit return rval while self._activeWakeups: w = self._activeWakeups.pop() if incomingHandler in (None, TransmitOnly): return w if not incomingHandler(w): return None return None
def testNoneComparedToZero(self): et1 = ExpiryTime(None) et2 = ExpiryTime(timedelta(days=0)) # None == forever, so it is greater than anything, although equal to itself self.assertGreater(et1, et2) self.assertLess(et2, et1) self.assertTrue(et1 >= et2) self.assertTrue(et2 <= et1)
def testNoneToUnExpiredComparison(self): et1 = ExpiryTime(None) et2 = ExpiryTime(timedelta(milliseconds=10)) self.assertNotEqual(et1, et2) self.assertNotEqual(et2, et1) sleep(et2.remainingSeconds()) self.assertNotEqual(et1, et2) self.assertNotEqual(et2, et1)
def testNoneToUnExpiredComparison(self): et1 = ExpiryTime(None) et2 = ExpiryTime(timedelta(milliseconds=10)) assert et1 != et2 assert et2 != et1 sleep(et2.remainingSeconds()) assert et1 != et2 assert et2 != et1
def testNoneComparedToZero(self): et1 = ExpiryTime(None) et2 = ExpiryTime(timedelta(days=0)) # None == forever, so it is greater than anything, although equal to itself assert et1 > et2 assert et2 < et1 assert et1 >= et2 assert et2 <= et1
def _update_remaining_hysteresis_period(self, reset=False): if not self._current_hysteresis: self._hysteresis_until = ExpiryTime(timedelta(seconds=0)) else: if reset or not self._hysteresis_until: self._hysteresis_until = ExpiryTime(self._current_hysteresis) else: self._hysteresis_until = ExpiryTime( self._current_hysteresis - self._hysteresis_until.remaining())
def testUnExpiredToUnExpiredComparison(self): et1 = ExpiryTime(timedelta(milliseconds=15)) et2 = ExpiryTime(timedelta(milliseconds=10)) assert et1 != et2 assert et2 != et1 sleep(et2.remainingSeconds()) print(str(et1), str(et2)) assert et1 != et2 assert et2 != et1 sleep(et1.remainingSeconds()) assert et1 == et2 assert et2 == et1
def testNoneComparedToNonZero(self): et1 = ExpiryTime(None) et2 = ExpiryTime(timedelta(milliseconds=10)) # None == forever, so it is greater than anything, although equal to itself self.assertGreater(et1, et2) self.assertLess(et2, et1) self.assertTrue(et1 > et2) self.assertTrue(et2 < et1) sleep(et2.remainingSeconds()) self.assertGreater(et1, et2) self.assertLess(et2, et1) self.assertTrue(et1 > et2) self.assertTrue(et2 < et1)
def testNoneComparedToNonZero(self): et1 = ExpiryTime(None) et2 = ExpiryTime(timedelta(milliseconds=10)) # None == forever, so it is greater than anything, although equal to itself assert et1 > et2 assert et2 < et1 assert et1 > et2 assert et2 < et1 sleep(et2.remainingSeconds()) assert et1 > et2 assert et2 < et1 assert et1 > et2 assert et2 < et1
def testNonZeroIsFalse(self): et = ExpiryTime(timedelta(milliseconds=10)) assert not et assert not bool(et) sleep(et.remainingSeconds()) assert et assert bool(et)
def shutdown(self): thesplog('ActorSystem shutdown requested.', level=logging.INFO) time_to_quit = ExpiryTime(MAX_SYSTEM_SHUTDOWN_DELAY) self.transport.scheduleTransmit( None, TransmitIntent(self.adminAddr, SystemShutdown(), onError=self._shutdownSendFailed)) while not time_to_quit.expired(): response = self.transport.run(None, time_to_quit.remaining()) if getattr(self, '_TASF', False): thesplog( 'Could not send shutdown request to Admin' '; aborting but not necessarily stopped', level=logging.WARNING) return if response: if isinstance(response.message, SystemShutdownCompleted): break else: thesplog('Expected shutdown completed message, got: %s', response.message, level=logging.WARNING) else: thesplog( 'No response to Admin shutdown request; Actor system not completely shutdown', level=logging.ERROR) thesplog('ActorSystem shutdown complete.')
def run(self): # Main loop for convention management. Wraps the lower-level # transport with a stop at the next needed convention # registration period to re-register. try: while not getattr(self, 'shutdown_completed', False): delay = min( self._cstate.convention_inattention_delay(), ExpiryTime(None) if self._hysteresisSender.delay.expired() else self._hysteresisSender.delay) # n.b. delay does not account for soon-to-expire # pingValids, but since delay will not be longer than # a CONVENTION_REREGISTRATION_PERIOD, the worst case # is a doubling of a pingValid period (which should be fine). r = self.transport.run(self.handleIncoming, delay.remaining()) # Check Convention status based on the elapsed time self._performIO(self._cstate.check_convention()) self._hysteresisSender.checkSends() except Exception as ex: import traceback thesplog('ActorAdmin uncaught exception: %s', traceback.format_exc(), level=logging.ERROR, exc_info=True) thesplog('Admin time to die', level=logging.DEBUG)
def sendWithHysteresis(self, intent): if self._hysteresis_until.expired(): self._current_hysteresis = self._hysteresis_min_period self._sender(intent) else: dups = self._keepIf( lambda M: (M.targetAddr != intent.targetAddr or type(M.message) != type(intent.message))) # The dups are duplicate sends to the new intent's target; complete them when # the actual message is finally sent with the same result if dups: intent.addCallback(self._dupSentGood(dups), self._dupSentFail(dups)) self._hysteresis_queue.append(intent) self._current_hysteresis = min( (self._hysteresis_min_period if (self._current_hysteresis is None or self._current_hysteresis < self._hysteresis_min_period) else self._current_hysteresis * self._hysteresis_rate), self._hysteresis_max_period) self._hysteresis_until = ExpiryTime( timedelta(seconds=0) if not self._current_hysteresis else ( self._current_hysteresis - (timedelta(seconds=0) if not self._hysteresis_until else self. _hysteresis_until.remaining())))
def loadActorSource(self, fname): self._LOADFAILED = None loadLimit = ExpiryTime(MAX_LOAD_SOURCE_DELAY) f = fname if hasattr(fname, 'read') else open(fname, 'rb') try: d = f.read() import hashlib hval = hashlib.md5(d).hexdigest() self.transport.scheduleTransmit( None, TransmitIntent( self.adminAddr, ValidateSource( hval, d, getattr( f, 'name', str(fname) if hasattr(fname, 'read') else fname)), onError=self._loadReqFailed)) while not loadLimit.expired(): if not self.transport.run(TransmitOnly, loadLimit.remaining()): break # all transmits completed if self._LOADFAILED or loadLimit.expired(): raise ActorSystemFailure('Load source failed due to ' + ( 'failure response (%s)' % self._LOADFAILED if self._LOADFAILED else 'timeout (%s)' % str(loadLimit))) return hval finally: f.close()
def setupConvention(self): if self.isShuttingDown(): return if not self._conventionAddress: gCA = getattr(self.transport, 'getConventionAddress', lambda c: None) self._conventionAddress = gCA(self.capabilities) if self._conventionAddress == self.myAddress: self._conventionAddress = None if self._conventionAddress: thesplog('Admin registering with Convention @ %s (%s)', self._conventionAddress, 'first time' if getattr(self, '_conventionLeaderIsGone', True) else 're-registering', level=logging.INFO, primary=True) self._hysteresisSender.sendWithHysteresis( TransmitIntent(self._conventionAddress, ConventionRegister( self.myAddress, self.capabilities, getattr(self, '_conventionLeaderIsGone', True)), onSuccess=self._setupConventionCBGood, onError=self._setupConventionCBError)) self._conventionRegistration = ExpiryTime( CONVENTION_REREGISTRATION_PERIOD)
def _remoteSystemCleanup(self, registrant): """Called when a RemoteActorSystem has exited and all associated Actors should be marked as exited and the ActorSystem removed from Convention membership. """ thesplog('Convention cleanup or deregistration for %s (new? %s)', registrant, registrant.actorAddressString not in self._conventionMembers, level=logging.INFO) if registrant.actorAddressString in self._conventionMembers: cmr = self._conventionMembers[registrant.actorAddressString] # Send exited notification to conventionNotificationHandler (if any) if self.isConventionLeader(): for each in self._conventionNotificationHandlers: self._send_intent( TransmitIntent( each, ActorSystemConventionUpdate( cmr.remoteAddress, cmr.remoteCapabilities, False))) # errors ignored # If the remote ActorSystem shutdown gracefully (i.e. sent # a Convention Deregistration) then it should not be # necessary to shutdown remote Actors (or notify of their # shutdown) because the remote ActorSystem should already # have caused this to occur. However, it won't hurt, and # it's necessary if the remote ActorSystem did not exit # gracefully. for lpa, raa in cmr.hasRemoteActors: # ignore errors: self._send_intent(TransmitIntent(lpa, ChildActorExited(raa))) # n.b. at present, this means that the parent might # get duplicate notifications of ChildActorExited; it # is expected that Actors can handle this. # Remove remote system from conventionMembers if not cmr.preRegistered: del self._conventionMembers[registrant.actorAddressString] else: # This conventionMember needs to stay because the # current system needs to continue issuing # registration pings. By setting the registryValid # expiration to forever, this member won't re-time-out # and will therefore be otherwise ignored... until it # registers again at which point the membership will # be updated with new settings. cmr.registryValid = ExpiryTime(None) if registrant == self._conventionAddress: # Convention Leader has exited. Do NOT set # conventionAddress to None. It might speed up shutdown # of this ActorSystem because it won't try to de-register # from the convention leader, but if the convention leader # reappears there will be nothing to get this ActorSystem # re-registered with the convention. self._conventionLeaderIsGone = True self._hysteresisSender.cancelSends(registrant)
def addWakeup(self, timePeriod): now = datetime.now() wakeupTime = now + timePeriod self._pendingWakeups.setdefault(wakeupTime, []) \ .append(ReceiveEnvelope(self.myAddress, WakeupMessage(timePeriod))) self.run_time = min([ExpiryTime(P - now) for P in self._pendingWakeups] + [self._max_runtime])
def testNonZeroIsFalse(self): et = ExpiryTime(timedelta(milliseconds=10)) self.assertFalse(et) self.assertFalse(bool(et)) sleep(et.remainingSeconds()) self.assertTrue(et) self.assertTrue(bool(et))
def __init__(self, address, capabilities): self.remoteAddress = address self.remoteCapabilities = capabilities self.registryValid = ExpiryTime(CONVENTION_REREGISTRATION_PERIOD * CONVENTION_REGISTRATION_MISS_MAX) self.hasRemoteActors = [ ] # (localParent, remoteActor) addresses created remotely self.lastMessaged = None # datetime of access; use with CONVENTION_HYSTERESIS_PERIOD
def __init__(self, *args, **kw): super(ConventioneerAdmin, self).__init__(*args, **kw) self._conventionMembers = { } # key=Remote Admin Addr, value=ConventionMemberData self._conventionRegistration = ExpiryTime(timedelta(seconds=0)) self._conventionNotificationHandlers = set() self._conventionAddress = None # Not a member; still could be a leader self._pendingSources = { } # key = sourceHash, value is array of PendingActor requests self._hysteresisSender = HysteresisDelaySender(self._send_intent)
def _reset_valid_timer(self): # registryValid is a timer that is usually set to a multiple # of the convention re-registration period. Each successful # convention re-registration resets the timer to the maximum # value (actually, it replaces this structure with a newly # generated structure). If the timer expires, the remote is # declared as dead and the registration is removed (or # quiesced if it is a pre-registration). self.registryValid = ExpiryTime(CONVENTION_REREGISTRATION_PERIOD * CONVENTION_REGISTRATION_MISS_MAX)
def __init__(self, actual_sender, hysteresis_min_period = HYSTERESIS_MIN_PERIOD, hysteresis_max_period = HYSTERESIS_MAX_PERIOD, hysteresis_rate = HYSTERESIS_RATE): self._sender = actual_sender self._hysteresis_until = ExpiryTime(timedelta(seconds=0)) self._hysteresis_queue = [] self._current_hysteresis = None # timedelta self._hysteresis_min_period = hysteresis_min_period self._hysteresis_max_period = hysteresis_max_period self._hysteresis_rate = hysteresis_rate
def run(self): try: while not self.isShuttingDown(): delay = min(self._conventionRegistration or \ ExpiryTime(CONVENTION_RESTART_PERIOD if self._conventionLost and not self.isConventionLeader() else CONVENTION_REREGISTRATION_PERIOD), ExpiryTime(None) if self._hysteresisSender.delay.expired() else self._hysteresisSender.delay ) self.transport.run(self.handleIncoming, delay.remaining()) self._checkConvention() self._hysteresisSender.checkSends() except Exception as ex: import traceback thesplog('ActorAdmin uncaught exception: %s', traceback.format_exc(), level=logging.ERROR, exc_info=True) thesplog('Admin time to die', level=logging.DEBUG)
def _remote_system_cleanup(self, registrant): """Called when a RemoteActorSystem has exited and all associated Actors should be marked as exited and the ActorSystem removed from Convention membership. This is also called on a First Time connection from the remote to discard any previous connection information. """ thesplog('Convention cleanup or deregistration for %s (known? %s)', registrant, bool(self._conventionMembers.find(registrant)), level=logging.INFO) rmsgs = [LostRemote(registrant)] cmr = self._conventionMembers.find(registrant) if cmr: # Send exited notification to conventionNotificationHandler (if any) for each in self._conventionNotificationHandlers: rmsgs.append( TransmitIntent( each, ActorSystemConventionUpdate(cmr.remoteAddress, cmr.remoteCapabilities, False))) # errors ignored # If the remote ActorSystem shutdown gracefully (i.e. sent # a Convention Deregistration) then it should not be # necessary to shutdown remote Actors (or notify of their # shutdown) because the remote ActorSystem should already # have caused this to occur. However, it won't hurt, and # it's necessary if the remote ActorSystem did not exit # gracefully. for lpa, raa in cmr.hasRemoteActors: # ignore errors: rmsgs.append(TransmitIntent(lpa, ChildActorExited(raa))) # n.b. at present, this means that the parent might # get duplicate notifications of ChildActorExited; it # is expected that Actors can handle this. # Remove remote system from conventionMembers if not cmr.preRegistered: self._conventionMembers.rmv(registrant) else: # This conventionMember needs to stay because the # current system needs to continue issuing # registration pings. By setting the registryValid # expiration to forever, this member won't re-time-out # and will therefore be otherwise ignored... until it # registers again at which point the membership will # be updated with new settings. cmr.registryValid = ExpiryTime(None) return rmsgs + [HysteresisCancel(registrant)]
def updateCapability(self, capabilityName, capabilityValue=None): self._updCAPFAILED = False attemptLimit = ExpiryTime(MAX_CAPABILITY_UPDATE_DELAY) self.transport.scheduleTransmit( None, TransmitIntent(self.adminAddr, CapabilityUpdate(capabilityName, capabilityValue), onError = self._updateCapsFailed)) while not attemptLimit.expired(): if not self.transport.run(TransmitOnly, attemptLimit.remaining()): break # all transmits completed if self._updCAPFAILED or attemptLimit.expired(): raise ActorSystemFailure("Could not update Actor System Admin capabilities.")
def __init__(self, myAddress, capabilities, sCBStats, getConventionAddressFunc): self._myAddress = myAddress self._capabilities = capabilities self._sCBStats = sCBStats self._conventionMembers = AssocList( ) # key=Remote Admin Addr, value=ConventionMemberData self._conventionNotificationHandlers = [] self._getConventionAddr = getConventionAddressFunc self._conventionAddress = getConventionAddressFunc(capabilities) self._conventionRegistration = ExpiryTime( CONVENTION_REREGISTRATION_PERIOD) self._has_been_activated = False
def check_convention(self): rmsgs = [] if not self._has_been_activated: return rmsgs if self.isConventionLeader() or not self.conventionLeaderAddr: missing = [] for each in self._conventionMembers.values(): if each.registryValid.expired(): missing.append(each) for each in missing: thesplog('%s missed %d checkins (%s); assuming it has died', str(each), CONVENTION_REGISTRATION_MISS_MAX, str(each.registryValid), level=logging.WARNING, primary=True) rmsgs.extend(self._remote_system_cleanup(each.remoteAddress)) self._conventionRegistration = ExpiryTime( CONVENTION_REREGISTRATION_PERIOD) else: # Re-register with the Convention if it's time if self.conventionLeaderAddr and self._conventionRegistration.expired( ): rmsgs.extend(self.setup_convention()) for member in self._conventionMembers.values(): if member.preRegistered and \ member.preRegistered.pingValid.expired() and \ not member.preRegistered.pingPending: member.preRegistered.pingPending = True member.preRegistered.pingValid = ExpiryTime( CONVENTION_RESTART_PERIOD if member.registryValid.expired( ) else CONVENTION_REREGISTRATION_PERIOD) rmsgs.append( HysteresisSend(member.remoteAddress, ConventionInvite(), onSuccess=self._preRegQueryNotPending, onError=self._preRegQueryNotPending)) return rmsgs