def loadActorSource(self, fname): self._LOADFAILED = None loadLimit = ExpiryTime(MAX_LOAD_SOURCE_DELAY) f = fname if hasattr(fname, 'read') else open(fname, 'rb') try: d = f.read() import hashlib hval = hashlib.md5(d).hexdigest() self.transport.scheduleTransmit( None, TransmitIntent( self.adminAddr, ValidateSource( hval, d, getattr( f, 'name', str(fname) if hasattr(fname, 'read') else fname)), onError=self._loadReqFailed)) while not loadLimit.expired(): if not self.transport.run(TransmitOnly, loadLimit.remaining()): break # all transmits completed if self._LOADFAILED or loadLimit.expired(): raise ActorSystemFailure('Load source failed due to ' + ( 'failure response (%s)' % self._LOADFAILED if self._LOADFAILED else 'timeout (%s)' % str(loadLimit))) return hval finally: f.close()
def updateCapability(self, capabilityName, capabilityValue=None): self._updCAPFAILED = False attemptLimit = ExpiryTime(MAX_CAPABILITY_UPDATE_DELAY) self.transport.scheduleTransmit( None, TransmitIntent(self.adminAddr, CapabilityUpdate(capabilityName, capabilityValue), onError = self._updateCapsFailed)) while not attemptLimit.expired(): if not self.transport.run(TransmitOnly, attemptLimit.remaining()): break # all transmits completed if self._updCAPFAILED or attemptLimit.expired(): raise ActorSystemFailure("Could not update Actor System Admin capabilities.")
def unloadActorSource(self, sourceHash): self._LOADFAILED = None loadLimit = ExpiryTime(MAX_LOAD_SOURCE_DELAY) self.transport.scheduleTransmit(None, TransmitIntent(self.adminAddr, ValidateSource(sourceHash, None), onError = self._loadReqFailed)) while not loadLimit.expired(): if not self.transport.run(TransmitOnly, loadLimit.remaining()): break # all transmits completed if self._LOADFAILED or loadLimit.expired(): raise ActorSystemFailure('Unload source failed due to ' + ('failure response' if self._LOADFAILED else 'timeout (%s)'%str(loadLimit)))
def shutdown(self): thesplog('ActorSystem shutdown requested.', level=logging.INFO) time_to_quit = ExpiryTime(MAX_SYSTEM_SHUTDOWN_DELAY) self.transport.scheduleTransmit( None, TransmitIntent(self.adminAddr, SystemShutdown(), onError=self._shutdownSendFailed)) while not time_to_quit.expired(): response = self.transport.run(None, time_to_quit.remaining()) if getattr(self, '_TASF', False): thesplog( 'Could not send shutdown request to Admin' '; aborting but not necessarily stopped', level=logging.WARNING) return if response: if isinstance(response.message, SystemShutdownCompleted): break else: thesplog('Expected shutdown completed message, got: %s', response.message, level=logging.WARNING) else: thesplog( 'No response to Admin shutdown request; Actor system not completely shutdown', level=logging.ERROR) thesplog('ActorSystem shutdown complete.')
def loadActorSource(self, fname): self._LOADFAILED = None loadLimit = ExpiryTime(MAX_LOAD_SOURCE_DELAY) f = fname if hasattr(fname, 'read') else open(fname, 'rb') try: d = f.read() import hashlib hval = hashlib.md5(d).hexdigest() self.transport.scheduleTransmit(None, TransmitIntent(self.adminAddr, ValidateSource(hval, d), onError = self._loadReqFailed)) while not loadLimit.expired(): if not self.transport.run(TransmitOnly, loadLimit.remaining()): break # all transmits completed if self._LOADFAILED or loadLimit.expired(): raise ActorSystemFailure('Load source failed due to ' + ('failure response (%s)'%self._LOADFAILED if self._LOADFAILED else 'timeout (%s)'%str(loadLimit))) return hval finally: f.close()
def __init__(self, system, logDefs=None): self._numPrimaries = 0 # Expects self.transport has already been set by subclass __init__ self.adminAddr = self.transport.getAdminAddr(system.capabilities) tryingTime = ExpiryTime(MAX_SYSTEM_SHUTDOWN_DELAY + timedelta(seconds=1)) while not tryingTime.expired(): if not self.transport.probeAdmin(self.adminAddr): self._startAdmin(self.adminAddr, self.transport.myAddress, system.capabilities, logDefs) if self._verifyAdminRunning(): return import time time.sleep(0.5) # Previous version may have been exiting if not self._verifyAdminRunning(): raise InvalidActorAddress( self.adminAddr, 'not a valid or useable ActorSystem Admin')
def __init__(self, system, logDefs = None): self._numPrimaries = 0 # Expects self.transport has already been set by subclass __init__ self.adminAddr = self.transport.getAdminAddr(system.capabilities) tryingTime = ExpiryTime(MAX_SYSTEM_SHUTDOWN_DELAY + timedelta(seconds=1)) while not tryingTime.expired(): if not self.transport.probeAdmin(self.adminAddr): self._startAdmin(self.adminAddr, self.transport.myAddress, system.capabilities, logDefs) if self._verifyAdminRunning(): return import time time.sleep(0.5) # Previous version may have been exiting if not self._verifyAdminRunning(): raise InvalidActorAddress(self.adminAddr, 'not a valid or useable ActorSystem Admin')
def tell(self, anActor, msg): attemptLimit = ExpiryTime(MAX_TELL_PERIOD) import socket for attempt in range(5000): try: self.transport.scheduleTransmit( None, TransmitIntent(anActor, msg, onError=self._tellFailed)) while not attemptLimit.expired(): if not self.transport.run(TransmitOnly, attemptLimit.remaining()): break # all transmits completed return except socket.error as ex: import errno if errno.EMFILE == ex.errno: import time time.sleep(0.1) else: raise
def shutdown(self): thesplog('ActorSystem shutdown requested.', level=logging.INFO) time_to_quit = ExpiryTime(MAX_SYSTEM_SHUTDOWN_DELAY) self.transport.scheduleTransmit( None, TransmitIntent(self.adminAddr, SystemShutdown(), onError=self._shutdownSendFailed)) while not time_to_quit.expired(): response = self.transport.run(None, time_to_quit.remaining()) if getattr(self, '_TASF', False): thesplog('Could not send shutdown request to Admin' '; aborting but not necessarily stopped', level=logging.WARNING) return if response: if isinstance(response.message, SystemShutdownCompleted): break else: thesplog('Expected shutdown completed message, got: %s', response.message, level=logging.WARNING) else: thesplog('No response to Admin shutdown request; Actor system not completely shutdown', level=logging.ERROR) thesplog('ActorSystem shutdown complete.')
def testNoneExpired(self): et = ExpiryTime(None) self.assertFalse(et.expired())
class HysteresisDelaySender(object): """Implements hysteresis delay for sending messages. This is intended to be used for messages exchanged between convention members to ensure that a mis-behaved member doesn't have the ability to inflict damage on the entire convention. The first time a message is sent via this sender it is passed on through, but that starts a blackout period that starts with the CONVENTION_HYSTERESIS_MIN_PERIOD. Each additional send attempt during that blackout period will cause the blackout period to be extended by the CONVENTION_HYSTERESIS_RATE, up to the CONVENTION_HYSTERESIS_MAX_PERIOD. Once the blackout period ends, the queued sends will be sent, but only the last attempted message of each type for the specified remote target. At that point, the hysteresis delay will be reduced by the CONVENTION_HYSTERESIS_RATE; further send attempts will affect the hysteresis blackout period as described as above but lack of sending attempts will continue to reduce the hysteresis back to a zero-delay setting. Note: delays are updated in a target-independent manner; the target is only considered when eliminating duplicates. Note: maxDelay on TransmitIntents is ignored by hysteresis delays. It is assumed that a transmit intent's maxDelay is greater than the maximum hysteresis period and/or that the hysteresis delay is more important than the transmit intent timeout. """ def __init__(self, actual_sender, hysteresis_min_period = HYSTERESIS_MIN_PERIOD, hysteresis_max_period = HYSTERESIS_MAX_PERIOD, hysteresis_rate = HYSTERESIS_RATE): self._sender = actual_sender self._hysteresis_until = ExpiryTime(timedelta(seconds=0)) self._hysteresis_queue = [] self._current_hysteresis = None # timedelta self._hysteresis_min_period = hysteresis_min_period self._hysteresis_max_period = hysteresis_max_period self._hysteresis_rate = hysteresis_rate @property def delay(self): return self._hysteresis_until def _has_hysteresis(self): return (self._current_hysteresis is not None and self._current_hysteresis >= self._hysteresis_min_period) def _increase_hysteresis(self): self._current_hysteresis = min( (self._current_hysteresis * self._hysteresis_rate) if self._has_hysteresis() else self._hysteresis_min_period, self._hysteresis_max_period) def _decrease_hysteresis(self): self._current_hysteresis = ( (self._current_hysteresis / self._hysteresis_rate) if self._has_hysteresis() else None) def _update_remaining_hysteresis_period(self, reset=False): if not self._current_hysteresis: self._hysteresis_until = ExpiryTime(timedelta(seconds=0)) else: if reset or not self._hysteresis_until: self._hysteresis_until = ExpiryTime(self._current_hysteresis) else: self._hysteresis_until = ExpiryTime( self._current_hysteresis - self._hysteresis_until.remaining()) def checkSends(self): if self.delay.expired(): self._decrease_hysteresis() self._update_remaining_hysteresis_period(reset=True) for intent in self._keepIf(lambda M: False): self._sender(intent) def sendWithHysteresis(self, intent): if self._hysteresis_until.expired(): self._current_hysteresis = self._hysteresis_min_period self._sender(intent) else: dups = self._keepIf(lambda M: (M.targetAddr != intent.targetAddr or type(M.message) != type(intent.message))) # The dups are duplicate sends to the new intent's target; complete them when # the actual message is finally sent with the same result if dups: intent.addCallback(self._dupSentGood(dups), self._dupSentFail(dups)) self._hysteresis_queue.append(intent) self._increase_hysteresis() self._update_remaining_hysteresis_period() def cancelSends(self, remoteAddr): for each in self._keepIf(lambda M: M.targetAddr != remoteAddr): each.tx_done(SendStatus.Failed) def _keepIf(self, keepFunc): requeues, removes = partition(keepFunc, self._hysteresis_queue) self._hysteresis_queue = requeues return removes @staticmethod def _dupSentGood(dups): def _finishDups(result, finishedIntent): for each in dups: each.tx_done(result) return _finishDups @staticmethod def _dupSentFail(dups): def _finishDups(result, finishedIntent): for each in dups: each.tx_done(result) return _finishDups
class ConventioneerAdmin(GlobalNamesAdmin): """Extends the AdminCore+GlobalNamesAdmin with ActorSystem Convention functionality to support multi-host configurations. """ def __init__(self, *args, **kw): super(ConventioneerAdmin, self).__init__(*args, **kw) self._conventionMembers = { } # key=Remote Admin Addr, value=ConventionMemberData self._conventionRegistration = ExpiryTime(timedelta(seconds=0)) self._conventionNotificationHandlers = set() self._conventionAddress = None # Not a member; still could be a leader self._pendingSources = { } # key = sourceHash, value is array of PendingActor requests self._hysteresisSender = HysteresisDelaySender(self._send_intent) def _updateStatusResponse(self, resp): resp.setConventionLeaderAddress(self._conventionAddress) resp.setConventionRegisterTime(self._conventionRegistration) for each in self._conventionMembers: resp.addConventioneer(self._conventionMembers[each].remoteAddress, self._conventionMembers[each].registryValid) resp.setNotifyHandlers(self._conventionNotificationHandlers) super(ConventioneerAdmin, self)._updateStatusResponse(resp) def _activate(self): self.setupConvention() @property def _conventionLost(self): "True if this system was part of a convention but are no longer" return getattr(self, '_conventionLeaderIsGone', False) def setupConvention(self): if self.isShuttingDown(): return if not self._conventionAddress: gCA = getattr(self.transport, 'getConventionAddress', lambda c: None) self._conventionAddress = gCA(self.capabilities) if self._conventionAddress == self.myAddress: self._conventionAddress = None if self._conventionAddress: thesplog('Admin registering with Convention @ %s (%s)', self._conventionAddress, 'first time' if getattr(self, '_conventionLeaderIsGone', True) else 're-registering', level=logging.INFO, primary=True) self._hysteresisSender.sendWithHysteresis( TransmitIntent(self._conventionAddress, ConventionRegister( self.myAddress, self.capabilities, getattr(self, '_conventionLeaderIsGone', True)), onSuccess=self._setupConventionCBGood, onError=self._setupConventionCBError)) self._conventionRegistration = ExpiryTime( CONVENTION_REREGISTRATION_PERIOD) def _setupConventionCBGood(self, result, finishedIntent): self._sCBStats.inc('Admin Convention Registered') self._conventionLeaderIsGone = False if hasattr(self, '_conventionLeaderMissCount'): delattr(self, '_conventionLeaderMissCount') def _setupConventionCBError(self, result, finishedIntent): self._sCBStats.inc('Admin Convention Registration Failed') if hasattr(self, '_conventionLeaderMissCount'): self._conventionLeaderMissCount += 1 else: self._conventionLeaderMissCount = 1 if self._conventionLeaderMissCount < CONVENTION_REGISTRATION_MISS_MAX: thesplog( 'Admin cannot register with convention @ %s (miss %d): %s', finishedIntent.targetAddr, self._conventionLeaderMissCount, result, level=logging.WARNING, primary=True) else: thesplog('Admin convention registration lost @ %s (miss %d): %s', finishedIntent.targetAddr, self._conventionLeaderMissCount, result, level=logging.ERROR, primary=True) if hasattr(self, '_conventionLeaderIsGone'): self._conventionLeaderIsGone = True def isConventionLeader(self): return self._conventionAddress is None or self._conventionAddress == self.myAddress def h_ConventionInvite(self, envelope): self._conventionAddress = envelope.sender self.setupConvention() def h_ConventionRegister(self, envelope): if self.isShuttingDown(): return self._sCBStats.inc('Admin Handle Convention Registration') # Registrant may re-register if changing capabilities registrant = envelope.message.adminAddress thesplog( 'Got Convention registration from %s (%s) (new? %s)', registrant, 'first time' if envelope.message.firstTime else 're-registering', registrant.actorAddressString not in self._conventionMembers, level=logging.INFO) if registrant == self.myAddress: # Either remote failed getting an external address and is # using 127.0.0.1 or else this is a malicious attempt to # make us talk to ourselves. Ignore it. thesplog( 'Convention registration from %s is an invalid address; ignoring.', registrant, level=logging.WARNING) return True if getattr(envelope.message, 'preRegister', False): # getattr used; see definition preReg = PreRegistration() # will attempt registration immediately else: preReg = (registrant.actorAddressString in self._conventionMembers and self._conventionMembers[ registrant.actorAddressString].preRegistered) if preReg: preReg.pingValid = ExpiryTime(CONVENTION_REREGISTRATION_PERIOD) #erase our knowledge of actors associated with potential former instance of this system if envelope.message.firstTime: self._remoteSystemCleanup(registrant) if registrant == self._conventionAddress: self._conventionLeaderIsGone = False newReg = registrant.actorAddressString not in self._conventionMembers self._conventionMembers[registrant.actorAddressString] = \ ConventionMemberData(registrant, envelope.message.capabilities) self._conventionMembers[ registrant.actorAddressString].preRegistered = preReg if self.isConventionLeader(): if newReg: for each in self._conventionNotificationHandlers: self._send_intent( TransmitIntent( each, ActorSystemConventionUpdate( registrant, envelope.message.capabilities, True))) # errors ignored # If we are the Convention Leader, this would be the point to # inform all other registrants of the new registrant. At # present, there is no reciprocity here, so just update the # new registrant with the leader's info. self._send_intent( TransmitIntent( registrant, ConventionRegister( self.myAddress, self.capabilities, # first time in, then must be first time out envelope.message.firstTime))) return True def h_ConventionDeRegister(self, envelope): self._sCBStats.inc('Admin Handle Convention De-registration') remoteAdmin = envelope.message.adminAddress if remoteAdmin == self.myAddress: # Either remote failed getting an external address and is # using 127.0.0.1 or else this is a malicious attempt to # make us talk to ourselves. Ignore it. thesplog( 'Convention deregistration from %s is an invalid address; ignoring.', remoteAdmin, level=logging.WARNING) return True if getattr(envelope.message, 'preRegistered', False): # see definition for getattr use if remoteAdmin.actorAddressString in self._conventionMembers: self._conventionMembers[ remoteAdmin.actorAddressString].preRegistered = None # Let normal timeout process complete the # deregistration; conversly, if the remote is still # active don't prematurely remove it. return True self._remoteSystemCleanup(remoteAdmin) return True def h_SystemShutdown(self, envelope): if self._conventionAddress: thesplog('Admin de-registering with Convention @ %s', str(self._conventionAddress), level=logging.INFO, primary=True) self._hysteresisSender.cancelSends(self._conventionAddress) self._send_intent( TransmitIntent(self._conventionAddress, ConventionDeRegister(self.myAddress))) return super(ConventioneerAdmin, self).h_SystemShutdown(envelope) def _remoteSystemCleanup(self, registrant): """Called when a RemoteActorSystem has exited and all associated Actors should be marked as exited and the ActorSystem removed from Convention membership. """ thesplog('Convention cleanup or deregistration for %s (new? %s)', registrant, registrant.actorAddressString not in self._conventionMembers, level=logging.INFO) if registrant.actorAddressString in self._conventionMembers: cmr = self._conventionMembers[registrant.actorAddressString] # Send exited notification to conventionNotificationHandler (if any) if self.isConventionLeader(): for each in self._conventionNotificationHandlers: self._send_intent( TransmitIntent( each, ActorSystemConventionUpdate( cmr.remoteAddress, cmr.remoteCapabilities, False))) # errors ignored # If the remote ActorSystem shutdown gracefully (i.e. sent # a Convention Deregistration) then it should not be # necessary to shutdown remote Actors (or notify of their # shutdown) because the remote ActorSystem should already # have caused this to occur. However, it won't hurt, and # it's necessary if the remote ActorSystem did not exit # gracefully. for lpa, raa in cmr.hasRemoteActors: # ignore errors: self._send_intent(TransmitIntent(lpa, ChildActorExited(raa))) # n.b. at present, this means that the parent might # get duplicate notifications of ChildActorExited; it # is expected that Actors can handle this. # Remove remote system from conventionMembers if not cmr.preRegistered: del self._conventionMembers[registrant.actorAddressString] else: # This conventionMember needs to stay because the # current system needs to continue issuing # registration pings. By setting the registryValid # expiration to forever, this member won't re-time-out # and will therefore be otherwise ignored... until it # registers again at which point the membership will # be updated with new settings. cmr.registryValid = ExpiryTime(None) if registrant == self._conventionAddress: # Convention Leader has exited. Do NOT set # conventionAddress to None. It might speed up shutdown # of this ActorSystem because it won't try to de-register # from the convention leader, but if the convention leader # reappears there will be nothing to get this ActorSystem # re-registered with the convention. self._conventionLeaderIsGone = True self._hysteresisSender.cancelSends(registrant) def _checkConvention(self): if self.isConventionLeader(): missing = [] for each in self._conventionMembers: if self._conventionMembers[each].registryValid.expired(): missing.append(each) for each in missing: thesplog('%s missed %d checkins (%s); assuming it has died', str(self._conventionMembers[each]), CONVENTION_REGISTRATION_MISS_MAX, str(self._conventionMembers[each].registryValid), level=logging.WARNING, primary=True) self._remoteSystemCleanup( self._conventionMembers[each].remoteAddress) self._conventionRegistration = ExpiryTime( CONVENTION_REREGISTRATION_PERIOD) else: # Re-register with the Convention if it's time if self._conventionAddress and self._conventionRegistration.expired( ): self.setupConvention() for each in self._conventionMembers: member = self._conventionMembers[each] if member.preRegistered and \ member.preRegistered.pingValid.expired() and \ not member.preRegistered.pingPending: member.preRegistered.pingPending = True member.preRegistered.pingValid = ExpiryTime( CONVENTION_RESTART_PERIOD if member.registryValid.expired( ) else CONVENTION_REREGISTRATION_PERIOD) self._hysteresisSender.sendWithHysteresis( TransmitIntent(member.remoteAddress, ConventionInvite(), onSuccess=self._preRegQueryNotPending, onError=self._preRegQueryNotPending)) def _preRegQueryNotPending(self, result, finishedIntent): remoteAddr = finishedIntent.targetAddr if remoteAddr.actorAddressString in self._conventionMembers: if self._conventionMembers[ remoteAddr.actorAddressString].preRegistered: self._conventionMembers[ remoteAddr. actorAddressString].preRegistered.pingPending = False def run(self): try: while not self.isShuttingDown(): delay = min(self._conventionRegistration or \ ExpiryTime(CONVENTION_RESTART_PERIOD if self._conventionLost and not self.isConventionLeader() else CONVENTION_REREGISTRATION_PERIOD), ExpiryTime(None) if self._hysteresisSender.delay.expired() else self._hysteresisSender.delay ) # n.b. delay does not account for soon-to-expire # pingValids, but since delay will not be longer than # a CONVENTION_REREGISTRATION_PERIOD, the worst case # is a doubling of a pingValid period (which should be fine). self.transport.run(self.handleIncoming, delay.remaining()) self._checkConvention() self._hysteresisSender.checkSends() except Exception as ex: import traceback thesplog('ActorAdmin uncaught exception: %s', traceback.format_exc(), level=logging.ERROR, exc_info=True) thesplog('Admin time to die', level=logging.DEBUG) # ---- Source Hash Transfers -------------------------------------------------- def h_SourceHashTransferRequest(self, envelope): sourceHash = envelope.message.sourceHash self._send_intent( TransmitIntent( envelope.sender, SourceHashTransferReply(sourceHash, self._sources.get(sourceHash, None)))) return True def h_SourceHashTransferReply(self, envelope): sourceHash = envelope.message.sourceHash pending = self._pendingSources[sourceHash] del self._pendingSources[sourceHash] if envelope.message.isValid(): self._sources[sourceHash] = envelope.message.sourceData for each in pending: self.h_PendingActor(each) else: for each in pending: self._sendPendingActorResponse( each, None, errorCode=PendingActorResponse.ERROR_Invalid_SourceHash) return True def h_ValidateSource(self, envelope): if not envelope.message.sourceData and envelope.sender != self._conventionAddress: # Propagate source unload requests to all convention members for each in self._conventionMembers: self._hysteresisSender.sendWithHysteresis( TransmitIntent(self._conventionMembers[each].remoteAddress, envelope.message)) super(ConventioneerAdmin, self).h_ValidateSource(envelope) return False # might have sent with hysteresis, so break out to local _run def _sentByRemoteAdmin(self, envelope): for each in self._conventionMembers: if envelope.sender == self._conventionMembers[each].remoteAddress: return True return False def _acceptsRemoteLoadedSourcesFrom(self, pendingActorEnvelope): allowed = self.capabilities.get('AllowRemoteActorSources', 'yes') return allowed.lower() == 'yes' or \ (allowed == 'LeaderOnly' and pendingActorEnvelope.sender == self._conventionAddress) # ---- Remote Actor interactions ---------------------------------------------- def h_PendingActor(self, envelope): sourceHash = envelope.message.sourceHash childRequirements = envelope.message.targetActorReq thesplog('Pending Actor request for %s%s reqs %s from %s', envelope.message.actorClassName, ' (%s)' % sourceHash if sourceHash else '', childRequirements, envelope.sender) # If this request was forwarded by a remote Admin and the # sourceHash is not known locally, request it from the sending # remote Admin if sourceHash and \ sourceHash not in self._sources and \ self._sentByRemoteAdmin(envelope) and \ self._acceptsRemoteLoadedSourcesFrom(envelope): requestedAlready = self._pendingSources.get(sourceHash, False) self._pendingSources.setdefault(sourceHash, []).append(envelope) if not requestedAlready: self._hysteresisSender.sendWithHysteresis( TransmitIntent(envelope.sender, SourceHashTransferRequest(sourceHash))) return False # sent with hysteresis, so break out to local _run return True # If the requested ActorClass is compatible with this # ActorSystem, attempt to start it, otherwise forward the # request to any known compatible ActorSystem. try: childClass = actualActorClass( envelope.message.actorClassName, partial(loadModuleFromHashSource, sourceHash, self._sources) if sourceHash else None) acceptsCaps = lambda caps: checkActorCapabilities( childClass, caps, childRequirements) if not acceptsCaps(self.capabilities): if envelope.message.forActor is None: # Request from external; use sender address envelope.message.forActor = envelope.sender remoteCandidates = [ K for K in self._conventionMembers if not self._conventionMembers[K].registryValid.expired() and self._conventionMembers[K].remoteAddress != envelope.sender # source Admin and self._conventionMembers[K].remoteAddress not in getattr(envelope.message, 'alreadyTried', []) and acceptsCaps(self._conventionMembers[K].remoteCapabilities) ] if not remoteCandidates: if self.isConventionLeader(): thesplog( 'No known ActorSystems can handle a %s for %s', childClass, envelope.message.forActor, level=logging.WARNING, primary=True) self._sendPendingActorResponse( envelope, None, errorCode=PendingActorResponse. ERROR_No_Compatible_ActorSystem) return True # Let the Convention Leader try to find an appropriate ActorSystem bestC = self._conventionAddress else: # distribute equally amongst candidates C = [(self._conventionMembers[K].remoteAddress, len(self._conventionMembers[K].hasRemoteActors)) for K in remoteCandidates] bestC = foldl( lambda best, possible: best if best[1] <= possible[1] else possible, C)[0] thesplog('Requesting creation of %s%s on remote admin %s', envelope.message.actorClassName, ' (%s)' % sourceHash if sourceHash else '', bestC) envelope.message.alreadyTried.append(self.myAddress) self._send_intent(TransmitIntent(bestC, envelope.message)) return True except InvalidActorSourceHash: self._sendPendingActorResponse( envelope, None, errorCode=PendingActorResponse.ERROR_Invalid_SourceHash) return True except InvalidActorSpecification: self._sendPendingActorResponse( envelope, None, errorCode=PendingActorResponse.ERROR_Invalid_ActorClass) return True except ImportError as ex: self._sendPendingActorResponse( envelope, None, errorCode=PendingActorResponse.ERROR_Import, errorStr=str(ex)) return True except AttributeError as ex: # Usually when the module has no attribute FooActor self._sendPendingActorResponse( envelope, None, errorCode=PendingActorResponse.ERROR_Invalid_ActorClass, errorStr=str(ex)) return True except Exception as ex: import traceback thesplog('Exception "%s" handling PendingActor: %s', ex, traceback.format_exc()) self._sendPendingActorResponse( envelope, None, errorCode=PendingActorResponse.ERROR_Invalid_ActorClass, errorStr=str(ex)) return True return super(ConventioneerAdmin, self).h_PendingActor(envelope) def h_NotifyOnSystemRegistration(self, envelope): if envelope.message.enableNotification: newRegistrant = envelope.sender not in self._conventionNotificationHandlers if newRegistrant: self._conventionNotificationHandlers.add(envelope.sender) # Now update the registrant on the current state of all convention members for member in self._conventionMembers: self._send_intent( TransmitIntent( envelope.sender, ActorSystemConventionUpdate( member, self._conventionMembers[member]. remoteCapabilities, True))) else: self._conventionNotificationHandlers.discard(envelope.sender) return True def h_PoisonMessage(self, envelope): self._conventionNotificationHandlers.discard(envelope.sender) def _handleChildExited(self, childAddr): self._conventionNotificationHandlers.discard(childAddr) return super(ConventioneerAdmin, self)._handleChildExited(childAddr) def h_CapabilityUpdate(self, envelope): updateLocals = self._updSystemCapabilities( envelope.message.capabilityName, envelope.message.capabilityValue) self.setupConvention() if updateLocals: self._capUpdateLocalActors() return False # might have sent with Hysteresis, so return to _run loop here
def testNonZeroExpired(self): et = ExpiryTime(timedelta(milliseconds=10)) self.assertFalse(et.expired()) sleep(et.remainingSeconds()) self.assertTrue(et.expired())
def testZeroExpired(self): et = ExpiryTime(timedelta(seconds=0)) self.assertTrue(et.expired())
class LocalConventionState(object): def __init__(self, myAddress, capabilities, sCBStats, getConventionAddressFunc): self._myAddress = myAddress self._capabilities = capabilities self._sCBStats = sCBStats self._conventionMembers = AssocList( ) # key=Remote Admin Addr, value=ConventionMemberData self._conventionNotificationHandlers = [] self._getConventionAddr = getConventionAddressFunc self._conventionAddress = getConventionAddressFunc(capabilities) self._conventionRegistration = ExpiryTime( CONVENTION_REREGISTRATION_PERIOD) self._has_been_activated = False @property def myAddress(self): return self._myAddress @property def capabilities(self): return self._capabilities def updateStatusResponse(self, resp): resp.setConventionLeaderAddress(self.conventionLeaderAddr) resp.setConventionRegisterTime(self._conventionRegistration) for each in self._conventionMembers.values(): resp.addConventioneer(each.remoteAddress, each.registryValid) resp.setNotifyHandlers(self._conventionNotificationHandlers) def active_in_convention(self): return self.conventionLeaderAddr and \ not self._conventionMembers.find(self.conventionLeaderAddr) @property def conventionLeaderAddr(self): return self._conventionAddress def isConventionLeader(self): return self.conventionLeaderAddr == self.myAddress def capabilities_have_changed(self, new_capabilities): self._capabilities = new_capabilities return self.setup_convention() def setup_convention(self, activation=False): self._has_been_activated |= activation rmsgs = [] self._conventionAddress = self._getConventionAddr(self.capabilities) leader_is_gone = ( self._conventionMembers.find(self.conventionLeaderAddr) is None ) if self.conventionLeaderAddr else True if not self.isConventionLeader() and self.conventionLeaderAddr: thesplog('Admin registering with Convention @ %s (%s)', self.conventionLeaderAddr, 'first time' if leader_is_gone else 're-registering', level=logging.INFO, primary=True) rmsgs.append( HysteresisSend(self.conventionLeaderAddr, ConventionRegister(self.myAddress, self.capabilities, leader_is_gone), onSuccess=self._setupConventionCBGood, onError=self._setupConventionCBError)) rmsgs.append(LogAggregator(self.conventionLeaderAddr)) self._conventionRegistration = ExpiryTime( CONVENTION_REREGISTRATION_PERIOD) return rmsgs def _setupConventionCBGood(self, result, finishedIntent): self._sCBStats.inc('Admin Convention Registered') if hasattr(self, '_conventionLeaderMissCount'): delattr(self, '_conventionLeaderMissCount') def _setupConventionCBError(self, result, finishedIntent): self._sCBStats.inc('Admin Convention Registration Failed') if hasattr(self, '_conventionLeaderMissCount'): self._conventionLeaderMissCount += 1 else: self._conventionLeaderMissCount = 1 if self._conventionLeaderMissCount < CONVENTION_REGISTRATION_MISS_MAX: thesplog( 'Admin cannot register with convention @ %s (miss %d): %s', finishedIntent.targetAddr, self._conventionLeaderMissCount, result, level=logging.WARNING, primary=True) else: thesplog('Admin convention registration lost @ %s (miss %d): %s', finishedIntent.targetAddr, self._conventionLeaderMissCount, result, level=logging.ERROR, primary=True) def got_convention_invite(self, sender): self._conventionAddress = sender return self.setup_convention() def got_convention_register(self, regmsg): self._sCBStats.inc('Admin Handle Convention Registration') # Registrant may re-register if changing capabilities rmsgs = [] registrant = regmsg.adminAddress prereg = getattr(regmsg, 'preRegister', False) # getattr used; see definition existing = self._conventionMembers.find(registrant) thesplog('Got Convention %sregistration from %s (%s) (new? %s)', 'pre-' if prereg else '', registrant, 'first time' if regmsg.firstTime else 're-registering', not existing, level=logging.INFO) if registrant == self.myAddress: # Either remote failed getting an external address and is # using 127.0.0.1 or else this is a malicious attempt to # make us talk to ourselves. Ignore it. thesplog( 'Convention registration from %s is an invalid address; ignoring.', registrant, level=logging.WARNING) return rmsgs #if prereg or (regmsg.firstTime and not prereg): if regmsg.firstTime or (prereg and not existing): # erase knowledge of actors associated with potential # former instance of this system existing = False rmsgs.extend(self._remote_system_cleanup(registrant)) if existing: existingPreReg = existing.preRegOnly existing.refresh(regmsg.capabilities, prereg) self._conventionMembers.add(registrant, existing) else: newmember = ConventionMemberData(registrant, regmsg.capabilities, prereg) if prereg: # Need to attempt registration immediately newmember.preRegistered = PreRegistration() self._conventionMembers.add(registrant, newmember) # Convention Members normally periodically initiate a # membership message, to which the leader confirms by # responding; if this was a pre-registration, that identifies # this system as the "leader" for that remote. Also, if the # remote sent this because it was a pre-registration leader, # it doesn't yet have all the member information so the member # should respond. #if self.isConventionLeader() or prereg or regmsg.firstTime: if self.isConventionLeader() or prereg or regmsg.firstTime or \ (existing and existing.preRegistered): if not existing or not existing.preRegOnly: # or prereg: first = prereg # If we are the Convention Leader, this would be the point to # inform all other registrants of the new registrant. At # present, there is no reciprocity here, so just update the # new registrant with the leader's info. rmsgs.append( TransmitIntent( registrant, ConventionRegister(self.myAddress, self.capabilities, first))) if (existing and existingPreReg and not existing.preRegOnly) or \ (not existing and newmember and not prereg): rmsgs.extend( self._notifications_of( ActorSystemConventionUpdate(registrant, regmsg.capabilities, True))) return rmsgs def _notifications_of(self, msg): return [ TransmitIntent(H, msg) for H in self._conventionNotificationHandlers ] def add_notification_handler(self, addr): if addr not in self._conventionNotificationHandlers: self._conventionNotificationHandlers.append(addr) # Now update the registrant on the current state of all convention members return [ TransmitIntent( addr, ActorSystemConventionUpdate(M.remoteAddress, M.remoteCapabilities, True)) for M in self._conventionMembers.values() ] return [] def remove_notification_handler(self, addr): self._conventionNotificationHandlers = [ H for H in self._conventionNotificationHandlers if H != addr ] def got_convention_deregister(self, deregmsg): self._sCBStats.inc('Admin Handle Convention De-registration') remoteAdmin = deregmsg.adminAddress if remoteAdmin == self.myAddress: # Either remote failed getting an external address and is # using 127.0.0.1 or else this is a malicious attempt to # make us talk to ourselves. Ignore it. thesplog( 'Convention deregistration from %s is an invalid address; ignoring.', remoteAdmin, level=logging.WARNING) if getattr(deregmsg, 'preRegistered', False): # see definition for getattr use existing = self._conventionMembers.find(remoteAdmin) if existing: existing.preRegistered = None return self._remote_system_cleanup(remoteAdmin) def got_system_shutdown(self): gen_ops = lambda addr: [ HysteresisCancel(addr), TransmitIntent(addr, ConventionDeRegister(self.myAddress)), ] if self.conventionLeaderAddr and \ self.conventionLeaderAddr != self.myAddress: thesplog('Admin de-registering with Convention @ %s', str(self.conventionLeaderAddr), level=logging.INFO, primary=True) return gen_ops(self.conventionLeaderAddr) return join( fmap(gen_ops, [ M.remoteAddress for M in self._conventionMembers.values() if M.remoteAddress != self.myAddress ])) def check_convention(self): rmsgs = [] if not self._has_been_activated: return rmsgs if self.isConventionLeader() or not self.conventionLeaderAddr: missing = [] for each in self._conventionMembers.values(): if each.registryValid.expired(): missing.append(each) for each in missing: thesplog('%s missed %d checkins (%s); assuming it has died', str(each), CONVENTION_REGISTRATION_MISS_MAX, str(each.registryValid), level=logging.WARNING, primary=True) rmsgs.extend(self._remote_system_cleanup(each.remoteAddress)) self._conventionRegistration = ExpiryTime( CONVENTION_REREGISTRATION_PERIOD) else: # Re-register with the Convention if it's time if self.conventionLeaderAddr and self._conventionRegistration.expired( ): rmsgs.extend(self.setup_convention()) for member in self._conventionMembers.values(): if member.preRegistered and \ member.preRegistered.pingValid.expired() and \ not member.preRegistered.pingPending: member.preRegistered.pingPending = True member.preRegistered.pingValid = ExpiryTime( CONVENTION_RESTART_PERIOD if member.registryValid.expired( ) else CONVENTION_REREGISTRATION_PERIOD) rmsgs.append( HysteresisSend(member.remoteAddress, ConventionInvite(), onSuccess=self._preRegQueryNotPending, onError=self._preRegQueryNotPending)) return rmsgs def _preRegQueryNotPending(self, result, finishedIntent): remoteAddr = finishedIntent.targetAddr member = self._conventionMembers.find(remoteAddr) if member and member.preRegistered: member.preRegistered.pingPending = False def _remote_system_cleanup(self, registrant): """Called when a RemoteActorSystem has exited and all associated Actors should be marked as exited and the ActorSystem removed from Convention membership. This is also called on a First Time connection from the remote to discard any previous connection information. """ thesplog('Convention cleanup or deregistration for %s (known? %s)', registrant, bool(self._conventionMembers.find(registrant)), level=logging.INFO) rmsgs = [LostRemote(registrant)] cmr = self._conventionMembers.find(registrant) if cmr: # Send exited notification to conventionNotificationHandler (if any) for each in self._conventionNotificationHandlers: rmsgs.append( TransmitIntent( each, ActorSystemConventionUpdate(cmr.remoteAddress, cmr.remoteCapabilities, False))) # errors ignored # If the remote ActorSystem shutdown gracefully (i.e. sent # a Convention Deregistration) then it should not be # necessary to shutdown remote Actors (or notify of their # shutdown) because the remote ActorSystem should already # have caused this to occur. However, it won't hurt, and # it's necessary if the remote ActorSystem did not exit # gracefully. for lpa, raa in cmr.hasRemoteActors: # ignore errors: rmsgs.append(TransmitIntent(lpa, ChildActorExited(raa))) # n.b. at present, this means that the parent might # get duplicate notifications of ChildActorExited; it # is expected that Actors can handle this. # Remove remote system from conventionMembers if not cmr.preRegistered: self._conventionMembers.rmv(registrant) else: # This conventionMember needs to stay because the # current system needs to continue issuing # registration pings. By setting the registryValid # expiration to forever, this member won't re-time-out # and will therefore be otherwise ignored... until it # registers again at which point the membership will # be updated with new settings. cmr.registryValid = ExpiryTime(None) return rmsgs + [HysteresisCancel(registrant)] def sentByRemoteAdmin(self, envelope): for each in self._conventionMembers.values(): if envelope.sender == each.remoteAddress: return True return False def convention_inattention_delay(self): return self._conventionRegistration or \ ExpiryTime(CONVENTION_REREGISTRATION_PERIOD if self.active_in_convention() or self.isConventionLeader() else CONVENTION_RESTART_PERIOD) def forward_pending_to_remote_system(self, childClass, envelope, sourceHash, acceptsCaps): alreadyTried = getattr(envelope.message, 'alreadyTried', []) remoteCandidates = [ K for K in self._conventionMembers.values() if not K.registryValid.expired() and K.remoteAddress != envelope.sender # source Admin and K.remoteAddress not in alreadyTried and acceptsCaps(K.remoteCapabilities) ] if not remoteCandidates: if self.isConventionLeader() or not self.conventionLeaderAddr: raise NoCompatibleSystemForActor( childClass, 'No known ActorSystems can handle a %s for %s', childClass, envelope.message.forActor) # Let the Convention Leader try to find an appropriate ActorSystem bestC = self.conventionLeaderAddr else: # distribute equally amongst candidates C = [(K.remoteAddress, len(K.hasRemoteActors)) for K in remoteCandidates] bestC = foldl( lambda best, possible: best if best[1] <= possible[1] else possible, C)[0] thesplog('Requesting creation of %s%s on remote admin %s', envelope.message.actorClassName, ' (%s)' % sourceHash if sourceHash else '', bestC) if bestC not in alreadyTried: # Don't send request to this remote again, it has already # been tried. This would also be indicated by that system # performing the add of self.myAddress as below, but if # there is disagreement between the local and remote # addresses, this addition will prevent continual # bounceback. alreadyTried.append(bestC) if self.myAddress not in alreadyTried: # Don't send request back to this actor system: it cannot # handle it alreadyTried.append(self.myAddress) envelope.message.alreadyTried = alreadyTried return [TransmitIntent(bestC, envelope.message)] def send_to_all_members(self, message, exception_list=None): return [ HysteresisSend(M.remoteAddress, message) for M in self._conventionMembers.values() if M.remoteAddress not in (exception_list or []) ]
def testNoneExpired(self): et = ExpiryTime(None) assert not et.expired()
def testNonZeroExpired(self): et = ExpiryTime(timedelta(milliseconds=10)) assert not et.expired() sleep(et.remainingSeconds()) assert et.expired()
def drainTransmits(self): drainLimit = ExpiryTime(MAX_SHUTDOWN_DRAIN_PERIOD) while not drainLimit.expired(): if not self.transport.run(TransmitOnly, drainLimit.remaining()): break # no transmits left
class ConventioneerAdmin(GlobalNamesAdmin): """Extends the AdminCore+GlobalNamesAdmin with ActorSystem Convention functionality to support multi-host configurations. """ def __init__(self, *args, **kw): super(ConventioneerAdmin, self).__init__(*args, **kw) self._conventionMembers = {} # key=Remote Admin Addr, value=ConventionMemberData self._conventionRegistration = ExpiryTime(timedelta(seconds=0)) self._conventionNotificationHandler = None self._conventionAddress = None # Not a member; still could be a leader self._pendingSources = {} # key = sourceHash, value is array of PendingActor requests self._hysteresisSender = HysteresisDelaySender(self._send_intent) def _updateStatusResponse(self, resp): resp.setConventionLeaderAddress(self._conventionAddress) resp.setConventionRegisterTime(self._conventionRegistration) for each in self._conventionMembers: resp.addConventioneer(self._conventionMembers[each].remoteAddress, self._conventionMembers[each].registryValid) resp.setNotifyHandler(self._conventionNotificationHandler) super(ConventioneerAdmin, self)._updateStatusResponse(resp) def _activate(self): self.setupConvention() @property def _conventionLost(self): "True if this system was part of a convention but are no longer" return getattr(self, '_conventionLeaderIsGone', False) def setupConvention(self): if self.isShuttingDown(): return if not self._conventionAddress: gCA = getattr(self.transport, 'getConventionAddress', lambda c: None) self._conventionAddress = gCA(self.capabilities) if self._conventionAddress == self.myAddress: self._conventionAddress = None if self._conventionAddress: thesplog('Admin registering with Convention @ %s (%s)', self._conventionAddress, 'first time' if getattr(self, '_conventionLeaderIsGone', True) else 're-registering', level=logging.INFO, primary=True) self._hysteresisSender.sendWithHysteresis( TransmitIntent(self._conventionAddress, ConventionRegister(self.myAddress, self.capabilities, getattr(self, '_conventionLeaderIsGone', True)), onSuccess = self._setupConventionCBGood, onError = self._setupConventionCBError)) self._conventionRegistration = ExpiryTime(CONVENTION_REREGISTRATION_PERIOD) def _setupConventionCBGood(self, result, finishedIntent): self._sCBStats.inc('Admin Convention Registered') self._conventionLeaderIsGone = False if hasattr(self, '_conventionLeaderMissCount'): delattr(self, '_conventionLeaderMissCount') def _setupConventionCBError(self, result, finishedIntent): self._sCBStats.inc('Admin Convention Registration Failed') if hasattr(self, '_conventionLeaderMissCount'): self._conventionLeaderMissCount += 1 else: self._conventionLeaderMissCount = 1 if self._conventionLeaderMissCount < CONVENTION_REGISTRATION_MISS_MAX: thesplog('Admin cannot register with convention @ %s (miss %d): %s', finishedIntent.targetAddr, self._conventionLeaderMissCount, result, level=logging.WARNING, primary=True) else: thesplog('Admin convention registration lost @ %s (miss %d): %s', finishedIntent.targetAddr, self._conventionLeaderMissCount, result, level=logging.ERROR, primary=True) if hasattr(self, '_conventionLeaderIsGone'): self._conventionLeaderIsGone = True def isConventionLeader(self): return self._conventionAddress is None or self._conventionAddress == self.myAddress def h_ConventionRegister(self, envelope): if self.isShuttingDown(): return self._sCBStats.inc('Admin Handle Convention Registration') # Registrant may re-register if changing capabilities registrant = envelope.message.adminAddress thesplog('Got Convention registration from %s (%s) (new? %s)', registrant, 'first time' if envelope.message.firstTime else 're-registering', registrant.actorAddressString not in self._conventionMembers, level=logging.INFO) if registrant == self.myAddress: # Either remote failed getting an external address and is # using 127.0.0.1 or else this is a malicious attempt to # make us talk to ourselves. Ignore it. thesplog('Convention registration from %s is an invalid address; ignoring.', registrant, level=logging.WARNING) return True #erase our knowledge of actors associated with potential former instance of this system if envelope.message.firstTime: self._remoteSystemCleanup(registrant) if registrant == self._conventionAddress: self._conventionLeaderIsGone = False newReg = registrant.actorAddressString not in self._conventionMembers self._conventionMembers[registrant.actorAddressString] = \ ConventionMemberData(registrant, envelope.message.capabilities) if self.isConventionLeader(): if newReg: if self._conventionNotificationHandler: self._send_intent( TransmitIntent(self._conventionNotificationHandler, ActorSystemConventionUpdate(registrant, envelope.message.capabilities, True))) # errors ignored # If we are the Convention Leader, this would be the point to # inform all other registrants of the new registrant. At # present, there is no reciprocity here, so just update the # new registrant with the leader's info. self._send_intent( TransmitIntent(registrant, ConventionRegister(self.myAddress, self.capabilities, # first time in, then must be first time out envelope.message.firstTime))) return True def h_ConventionDeRegister(self, envelope): self._sCBStats.inc('Admin Handle Convention De-registration') remoteAdmin = envelope.message.adminAddress if remoteAdmin == self.myAddress: # Either remote failed getting an external address and is # using 127.0.0.1 or else this is a malicious attempt to # make us talk to ourselves. Ignore it. thesplog('Convention deregistration from %s is an invalid address; ignoring.', remoteAdmin, level=logging.WARNING) return True self._remoteSystemCleanup(remoteAdmin) return True def h_SystemShutdown(self, envelope): if self._conventionAddress: thesplog('Admin de-registering with Convention @ %s', str(self._conventionAddress), level=logging.INFO, primary=True) self._hysteresisSender.cancelSends(self._conventionAddress) self._send_intent( TransmitIntent(self._conventionAddress, ConventionDeRegister(self.myAddress))) return super(ConventioneerAdmin, self).h_SystemShutdown(envelope) def _remoteSystemCleanup(self, registrant): """Called when a RemoteActorSystem has exited and all associated Actors should be marked as exited and the ActorSystem removed from Convention membership. """ thesplog('Convention cleanup or deregistration for %s (new? %s)', registrant, registrant.actorAddressString not in self._conventionMembers, level=logging.INFO) if registrant.actorAddressString in self._conventionMembers: cmr = self._conventionMembers[registrant.actorAddressString] # Send exited notification to conventionNotificationHandler (if any) if self.isConventionLeader() and self._conventionNotificationHandler: self._send_intent( TransmitIntent(self._conventionNotificationHandler, ActorSystemConventionUpdate(cmr.remoteAddress, cmr.remoteCapabilities, False))) # errors ignored # If the remote ActorSystem shutdown gracefully (i.e. sent # a Convention Deregistration) then it should not be # necessary to shutdown remote Actors (or notify of their # shutdown) because the remote ActorSystem should already # have caused this to occur. However, it won't hurt, and # it's necessary if the remote ActorSystem did not exit # gracefully. for lpa, raa in cmr.hasRemoteActors: # ignore errors: self._send_intent(TransmitIntent(lpa, ChildActorExited(raa))) # n.b. at present, this means that the parent might # get duplicate notifications of ChildActorExited; it # is expected that Actors can handle this. # Remove remote system from conventionMembers del self._conventionMembers[registrant.actorAddressString] if registrant == self._conventionAddress: # Convention Leader has exited. Do NOT set # conventionAddress to None. It might speed up shutdown # of this ActorSystem because it won't try to de-register # from the convention leader, but if the convention leader # reappears there will be nothing to get this ActorSystem # re-registered with the convention. self._conventionLeaderIsGone = True self._hysteresisSender.cancelSends(registrant) def _checkConvention(self): if self.isConventionLeader(): missing = [] for each in self._conventionMembers: if self._conventionMembers[each].registryValid.expired(): missing.append(each) for each in missing: thesplog('%s missed %d checkins (%s); assuming it has died', str(self._conventionMembers[each]), CONVENTION_REGISTRATION_MISS_MAX, str(self._conventionMembers[each].registryValid), level=logging.WARNING, primary=True) self._remoteSystemCleanup(self._conventionMembers[each].remoteAddress) self._conventionRegistration = ExpiryTime(CONVENTION_REREGISTRATION_PERIOD) else: # Re-register with the Convention if it's time if self._conventionAddress and self._conventionRegistration.expired(): self.setupConvention() def run(self): try: while not self.isShuttingDown(): delay = min(self._conventionRegistration or \ ExpiryTime(CONVENTION_RESTART_PERIOD if self._conventionLost and not self.isConventionLeader() else CONVENTION_REREGISTRATION_PERIOD), ExpiryTime(None) if self._hysteresisSender.delay.expired() else self._hysteresisSender.delay ) self.transport.run(self.handleIncoming, delay.remaining()) self._checkConvention() self._hysteresisSender.checkSends() except Exception as ex: import traceback thesplog('ActorAdmin uncaught exception: %s', traceback.format_exc(), level=logging.ERROR, exc_info=True) thesplog('Admin time to die', level=logging.DEBUG) # ---- Source Hash Transfers -------------------------------------------------- def h_SourceHashTransferRequest(self, envelope): sourceHash = envelope.message.sourceHash self._send_intent( TransmitIntent(envelope.sender, SourceHashTransferReply(sourceHash, self._sources.get(sourceHash, None)))) return True def h_SourceHashTransferReply(self, envelope): sourceHash = envelope.message.sourceHash pending = self._pendingSources[sourceHash] del self._pendingSources[sourceHash] if envelope.message.isValid(): self._sources[sourceHash] = envelope.message.sourceData for each in pending: self.h_PendingActor(each) else: for each in pending: self._sendPendingActorResponse(each, None, errorCode = PendingActorResponse.ERROR_Invalid_SourceHash) return True def h_ValidateSource(self, envelope): if not envelope.message.sourceData and envelope.sender != self._conventionAddress: # Propagate source unload requests to all convention members for each in self._conventionMembers: self._hysteresisSender.sendWithHysteresis( TransmitIntent(self._conventionMembers[each].remoteAddress, envelope.message)) super(ConventioneerAdmin, self).h_ValidateSource(envelope) return False # might have sent with hysteresis, so break out to local _run def _sentByRemoteAdmin(self, envelope): for each in self._conventionMembers: if envelope.sender == self._conventionMembers[each].remoteAddress: return True return False def _acceptsRemoteLoadedSourcesFrom(self, pendingActorEnvelope): allowed = self.capabilities.get('AllowRemoteActorSources', 'yes') return allowed.lower() == 'yes' or \ (allowed == 'LeaderOnly' and pendingActorEnvelope.sender == self._conventionAddress) def h_PendingActor(self, envelope): sourceHash = envelope.message.sourceHash childRequirements = envelope.message.targetActorReq # If this request was forwarded by a remote Admin and the # sourceHash is not known locally, request it from the sending # remote Admin if sourceHash and \ sourceHash not in self._sources and \ self._sentByRemoteAdmin(envelope) and \ self._acceptsRemoteLoadedSourcesFrom(envelope): requestedAlready = self._pendingSources.get(sourceHash, False) self._pendingSources.setdefault(sourceHash, []).append(envelope) if not requestedAlready: self._hysteresisSender.sendWithHysteresis( TransmitIntent(envelope.sender, SourceHashTransferRequest(sourceHash))) return False # sent with hysteresis, so break out to local _run return True # If the requested ActorClass is compatible with this # ActorSystem, attempt to start it, otherwise forward the # request to any known compatible ActorSystem. try: childClass = actualActorClass(envelope.message.actorClassName, partial(loadModuleFromHashSource, sourceHash, self._sources) if sourceHash else None) acceptsCaps = lambda caps: checkActorCapabilities(childClass, caps, childRequirements) if not acceptsCaps(self.capabilities): if envelope.message.forActor is None: # Request from external; use sender address envelope.message.forActor = envelope.sender remoteCandidates = [ K for K in self._conventionMembers if not self._conventionMembers[K].registryValid.expired() and self._conventionMembers[K].remoteAddress != envelope.sender # source Admin and acceptsCaps(self._conventionMembers[K].remoteCapabilities)] if not remoteCandidates: if self.isConventionLeader(): thesplog('No known ActorSystems can handle a %s for %s', childClass, envelope.message.forActor, level=logging.WARNING, primary=True) self._sendPendingActorResponse(envelope, None, errorCode = PendingActorResponse.ERROR_No_Compatible_ActorSystem) return True # Let the Convention Leader try to find an appropriate ActorSystem bestC = self._conventionAddress else: # distribute equally amongst candidates C = [(self._conventionMembers[K].remoteAddress, len(self._conventionMembers[K].hasRemoteActors)) for K in remoteCandidates] bestC = foldl(lambda best,possible: best if best[1] <= possible[1] else possible, C)[0] self._send_intent(TransmitIntent(bestC, envelope.message)) return True except InvalidActorSourceHash: self._sendPendingActorResponse(envelope, None, errorCode = PendingActorResponse.ERROR_Invalid_SourceHash) return True except InvalidActorSpecification: self._sendPendingActorResponse(envelope, None, errorCode = PendingActorResponse.ERROR_Invalid_ActorClass) return True except ImportError: self._sendPendingActorResponse(envelope, None, errorCode = PendingActorResponse.ERROR_Import) return True return super(ConventioneerAdmin, self).h_PendingActor(envelope) def h_NotifyOnSystemRegistration(self, envelope): if envelope.message.enableNotification: self._conventionNotificationHandler = envelope.sender # Now update the registrant on the current state of all convention members for member in self._conventionMembers: self._send_intent( TransmitIntent(envelope.sender, ActorSystemConventionUpdate(member, self._conventionMembers[member].remoteCapabilities, True))) else: if envelope.sender == self._conventionNotificationHandler: self._conventionNotificationHandler = None return True def h_CapabilityUpdate(self, envelope): updateLocals = self._updSystemCapabilities( envelope.message.capabilityName, envelope.message.capabilityValue) self.setupConvention() if updateLocals: self._capUpdateLocalActors() return False # might have sent with Hysteresis, so return to _run loop here
class wakeupTransportBase(object): """The wakeupTransportBase is designed to be used as a mixin-base for a Transport class and provides handling for the wakeupAfter() functionality. This base mixin provides the primary .run() entrypoint for the transport and a .run_time ExpiryTime member that provides the remaining time-to-run period. The system can handle .wakeupAfter() requests by calling this class's .addWakeup() method with the datetime.timedelta for the wakeup to be scheduled. The Transport should provide the following: ._runWithExpiry(incomingHandler) Called by this class's .run() entrypoint to do the actual transport-specific run routine. Should perform that activity while the self.run_time ExpiryTime is not expired (self.run_time will be updated when new wakeupAfter() events are scheduled). """ def __init__(self, *args, **kw): super(wakeupTransportBase, self).__init__(*args, **kw) # _pendingWakeups: key = datetime for wakeup, value = list of # pending wakeupAfter msgs to restart at that time self._pendingWakeups = {} self._activeWakeups = [] # expired wakeups to be delivered def _updateStatusResponse(self, resp): "Called to update a Thespian_SystemStatus or Thespian_ActorStatus with common information" resp.addWakeups(self._pendingWakeups) for each in self._activeWakeups: resp.addPendingMessage(self.myAddress, self.myAddress, str(each.message)) def run(self, incomingHandler, maximumDuration=None): """Core scheduling method; called by the current Actor process when idle to await new messages (or to do background processing). """ self._max_runtime = ExpiryTime(maximumDuration) while not self._max_runtime.expired(): now = datetime.now() self.run_time = min([ExpiryTime(P - now) for P in self._pendingWakeups] + [self._max_runtime]) rval = self._runWithExpiry(incomingHandler) if rval is not None: return rval if not self._realizeWakeups(): # No wakeups were processed, and the inner run # returned, so assume there's nothing to do and exit return rval while self._activeWakeups: w = self._activeWakeups.pop() if incomingHandler in (None, TransmitOnly): return w if not incomingHandler(w): return None return None def addWakeup(self, timePeriod): now = datetime.now() wakeupTime = now + timePeriod self._pendingWakeups.setdefault(wakeupTime, []) \ .append(ReceiveEnvelope(self.myAddress, WakeupMessage(timePeriod))) self.run_time = min([ExpiryTime(P - now) for P in self._pendingWakeups] + [self._max_runtime]) def _realizeWakeups(self): "Find any expired wakeups and queue them to the send processing queue" now = datetime.now() removals = [] for wakeupTime in self._pendingWakeups: if wakeupTime > now: continue self._activeWakeups.extend(self._pendingWakeups[wakeupTime]) removals.append(wakeupTime) for each in removals: del self._pendingWakeups[each] return bool(removals)
class HysteresisDelaySender(object): """Implements hysteresis delay for sending messages. This is intended to be used for messages exchanged between convention members to ensure that a mis-behaved member doesn't have the ability to inflict damage on the entire convention. The first time a message is sent via this sender it is passed on through, but that starts a blackout period that starts with the CONVENTION_HYSTERESIS_MIN_PERIOD. Each additional send attempt during that blackout period will cause the blackout period to be extended by the CONVENTION_HYSTERESIS_RATE, up to the CONVENTION_HYSTERESIS_MAX_PERIOD. Once the blackout period ends, the queued sends will be sent, but only the last attempted message of each type for the specified remote target. At that point, the hysteresis delay will be reduced by the CONVENTION_HYSTERESIS_RATE; further send attempts will affect the hysteresis blackout period as described as above but lack of sending attempts will continue to reduce the hysteresis back to a zero-delay setting. Note: delays are updated in a target-independent manner; the target is only considered when eliminating duplicates. Note: maxDelay on TransmitIntents is ignored by hysteresis delays. It is assumed that a transmit intent's maxDelay is greater than the maximum hysteresis period and/or that the hysteresis delay is more important than the transmit intent timeout. """ def __init__(self, actual_sender, hysteresis_min_period = HYSTERESIS_MIN_PERIOD, hysteresis_max_period = HYSTERESIS_MAX_PERIOD, hysteresis_rate = HYSTERESIS_RATE): self._sender = actual_sender self._hysteresis_until = ExpiryTime(timedelta(seconds=0)) self._hysteresis_queue = [] self._duplicate_queue = [] self._current_hysteresis = None # timedelta self._hysteresis_min_period = hysteresis_min_period self._hysteresis_max_period = hysteresis_max_period self._hysteresis_rate = hysteresis_rate @property def delay(self): return self._hysteresis_until def checkSends(self): if self.delay.expired(): hsends = self._hysteresis_queue self._hysteresis_queue = [] self._current_hysteresis = ( None if (self._current_hysteresis is None or self._current_hysteresis < self._hysteresis_min_period) else self._current_hysteresis / self._hysteresis_rate) self._hysteresis_until = ExpiryTime(self._current_hysteresis if self._current_hysteresis else timedelta(seconds=0)) for intent in hsends: self._sender(intent) def sendWithHysteresis(self, intent): if self._hysteresis_until.expired(): self._current_hysteresis = self._hysteresis_min_period self._sender(intent) else: dups = self._keepIf(lambda M: (M.targetAddr != intent.targetAddr or type(M.message) != type(intent.message))) # The dups are duplicate sends to the new intent's target; complete them when # the actual message is finally sent with the same result if dups: intent.addCallback(self._dupSentGood(dups), self._dupSentFail(dups)) self._hysteresis_queue.append(intent) self._current_hysteresis = min( (self._hysteresis_min_period if (self._current_hysteresis is None or self._current_hysteresis < self._hysteresis_min_period) else self._current_hysteresis * self._hysteresis_rate), self._hysteresis_max_period) self._hysteresis_until = ExpiryTime( timedelta(seconds=0) if not self._current_hysteresis else (self._current_hysteresis - (timedelta(seconds=0) if not self._hysteresis_until else self._hysteresis_until.remaining()))) def cancelSends(self, remoteAddr): cancels = self._keepIf(lambda M: M.targetAddr != remoteAddr) for each in cancels: each.result = SendStatus.Failed each.completionCallback() def _keepIf(self, keepFunc): requeues, removes = partition(keepFunc, self._hysteresis_queue) self._hysteresis_queue = requeues return removes @staticmethod def _dupSentGood(dups): def _finishDups(result, finishedIntent): for each in dups: each.result = result each.completionCallback() return _finishDups @staticmethod def _dupSentFail(dups): def _finishDups(result, finishedIntent): for each in dups: each.result = result each.completionCallback() return _finishDups
class wakeupTransportBase(object): """The wakeupTransportBase is designed to be used as a mixin-base for a Transport class and provides handling for the wakeupAfter() functionality. This base mixin provides the primary .run() entrypoint for the transport and a .run_time ExpiryTime member that provides the remaining time-to-run period. The system can handle .wakeupAfter() requests by calling this class's .addWakeup() method with the datetime.timedelta for the wakeup to be scheduled. The Transport should provide the following: ._runWithExpiry(incomingHandler) Called by this class's .run() entrypoint to do the actual transport-specific run routine. Should perform that activity while the self.run_time ExpiryTime is not expired (self.run_time will be updated when new wakeupAfter() events are scheduled). """ def __init__(self, *args, **kw): super(wakeupTransportBase, self).__init__(*args, **kw) # _pendingWakeups: key = datetime for wakeup, value = list of # pending wakeupAfter msgs to restart at that time self._pendingWakeups = {} self._activeWakeups = [] # expired wakeups to be delivered def _updateStatusResponse(self, resp): "Called to update a Thespian_SystemStatus or Thespian_ActorStatus with common information" resp.addWakeups(self._pendingWakeups) for each in self._activeWakeups: resp.addPendingMessage(self.myAddress, self.myAddress, str(each.message)) def run(self, incomingHandler, maximumDuration=None): """Core scheduling method; called by the current Actor process when idle to await new messages (or to do background processing). """ self._max_runtime = ExpiryTime(maximumDuration) while not self._max_runtime.expired(): now = datetime.now() self.run_time = min( [ExpiryTime(P - now) for P in self._pendingWakeups] + [self._max_runtime]) rval = self._runWithExpiry(incomingHandler) if rval is not None: return rval if not self._realizeWakeups(): # No wakeups were processed, and the inner run # returned, so assume there's nothing to do and exit return rval while self._activeWakeups: w = self._activeWakeups.pop() if incomingHandler is None: return w if not incomingHandler(w): return None return None def addWakeup(self, timePeriod): now = datetime.now() wakeupTime = now + timePeriod self._pendingWakeups.setdefault(wakeupTime, []) \ .append(ReceiveEnvelope(self.myAddress, WakeupMessage(timePeriod))) self.run_time = min( [ExpiryTime(P - now) for P in self._pendingWakeups] + [self._max_runtime]) def _realizeWakeups(self): "Find any expired wakeups and queue them to the send processing queue" now = datetime.now() removals = [] for wakeupTime in self._pendingWakeups: if wakeupTime > now: continue self._activeWakeups.extend(self._pendingWakeups[wakeupTime]) removals.append(wakeupTime) for each in removals: del self._pendingWakeups[each] return bool(removals)
def testZeroExpired(self): et = ExpiryTime(timedelta(seconds=0)) assert et.expired()