class MultiprocessQueueTCore_Common(object): def __init__(self, myQueue, parentQ, adminQ, adminAddr): self._myInputQ = myQueue self._parentQ = parentQ self._adminQ = adminQ self._adminAddr = adminAddr # _queues is a map of direct child ActorAddresses to Queue instance. Note # that there will be multiple keys mapping to the same Queue # instance because routing is only either to the Parent or to # an immediate Child. self._queues = AssocList() # addr -> queue # _fwdvia represents routing for other than immediate parent # or child (there may be multiple target addresses mapping to # the same forward address. self._fwdvia = AssocList() # targetAddress -> fwdViaAddress self._deadaddrs = [] # Signals can set these to true; they should be checked and # reset by the main processing loop. There is a small window # where they could be missed because signals are not queued, # but this should handle the majority of situations. Note # that the Queue object is NOT signal-safe, so don't try to # queue signals that way. self._checkChildren = False self._shutdownSignalled = False def mainLocalInputQueueEndpoint(self): return self._myInputQ def adminQueueEndpoint(self): return self._adminQ @property def adminAddr(self): return self._adminAddr def protectedFileNumList(self): return foldl(lambda a, b: a+[b._reader.fileno(), b._writer.fileno()], [self._myInputQ, self._parentQ, self._adminQ] + list(self._queues.values()), []) def childResetFileNumList(self): return foldl(lambda a, b: a+[b._reader.fileno(), b._writer.fileno()], [self._parentQ] + list(self._queues.values()), []) def add_endpoint(self, child_addr, child_queue): self._queues.add(child_addr, child_queue) def set_address_to_dead(self, child_addr): self._queues.rmv(child_addr) self._fwdvia.rmv(child_addr) self._fwdvia.rmv_value(child_addr) self._deadaddrs.append(child_addr) def abort_core_run(self): self._aborting_run = True def core_common_transmit(self, transmit_intent, from_addr): try: if self.isMyAddress(transmit_intent.targetAddr): if transmit_intent.message: self._myInputQ.put( (from_addr, transmit_intent.serMsg), True, timePeriodSeconds(transmit_intent.delay())) else: tgtQ = self._queues.find(transmit_intent.targetAddr) if tgtQ: tgtQ.put((from_addr, transmit_intent.serMsg), True, timePeriodSeconds(transmit_intent.delay())) else: # None means sent by parent, so don't send BACK to parent if unknown topOrFromBelow = from_addr if self._parentQ else None (self._parentQ or self._adminQ).put( (topOrFromBelow, transmit_intent.serMsg), True, timePeriodSeconds(transmit_intent.delay())) transmit_intent.tx_done(SendStatus.Sent) return except Q.Full: pass transmit_intent.tx_done(SendStatus.DeadTarget if not isinstance( transmit_intent._message, (ChildActorExited, ActorExitRequest)) else SendStatus.Failed) def core_common_receive(self, incoming_handler, for_local_addr, run_time_f): """Core scheduling method; called by the current Actor process when idle to await new messages (or to do background processing). """ if incoming_handler == TransmitOnly or \ isinstance(incoming_handler, TransmitOnly): # transmits are not queued/multistage in this transport, no waiting return 0 self._aborting_run = False while not run_time_f().expired() and not self._aborting_run: try: # Unfortunately, the Queue object is not signal-safe, # so a frequent wakeup is needed to check # _checkChildren and _shutdownSignalled. rcvd = self._myInputQ.get(True, min(run_time_f().remainingSeconds() or QUEUE_CHECK_PERIOD, QUEUE_CHECK_PERIOD)) except Q.Empty: if not self._checkChildren and not self._shutdownSignalled: # Probably a timeout, but let the while loop decide for sure continue rcvd = 'BuMP' if rcvd == 'BuMP': relayAddr = sendAddr = destAddr = for_local_addr if self._checkChildren: self._checkChildren = False msg = ChildMayHaveDied() elif self._shutdownSignalled: self._shutdownSignalled = False msg = ActorExitRequest() else: return Thespian__UpdateWork() else: relayAddr, (sendAddr, destAddr, msg) = rcvd if not self._queues.find(sendAddr): # We don't directly know about this sender, so # remember what path this arrived on to know where to # direct future messages for this sender. if relayAddr and self._queues.find(relayAddr) and \ not self._fwdvia.find(sendAddr): # relayAddr might be None if it's our parent, which is OK because # the default message forwarding is to the parent. If it's not # none, it should be in self._queues though! self._fwdvia.add(sendAddr, relayAddr) if hasattr(self, '_addressMgr'): destAddr,msg = self._addressMgr.prepMessageSend(destAddr, msg) if destAddr is None: thesplog('Unexpected target inaccessibility for %s', msg, level = logging.WARNING) raise CannotPickleAddress(destAddr) if msg is SendStatus.DeadTarget: thesplog('Faking message "sent" because target is dead and recursion avoided.') continue if self.isMyAddress(destAddr): if incoming_handler is None: return ReceiveEnvelope(sendAddr, msg) r = incoming_handler(ReceiveEnvelope(sendAddr, msg)) if not r: return r # handler returned False, indicating run() should exit else: # Note: the following code has implicit knowledge of serialize() and xmit putQValue = lambda relayer: (relayer, (sendAddr, destAddr, msg)) deadQValue = lambda relayer: (relayer, (sendAddr, self._adminAddr, DeadEnvelope(destAddr, msg))) # Must forward this packet via a known forwarder or our parent. send_dead = False tgtQ = self._queues.find(destAddr) if tgtQ: sendArgs = putQValue(for_local_addr), True if not tgtQ: tgtA = self._fwdvia.find(destAddr) if tgtA: tgtQ = self._queues.find(tgtA) sendArgs = putQValue(None), else: for each in self._deadaddrs: if destAddr == each: send_dead = True if tgtQ: try: tgtQ.put(*sendArgs, timeout=timePeriodSeconds(MAX_QUEUE_TRANSMIT_PERIOD)) continue except Q.Full: thesplog('Unable to send msg %s to dest %s; dead lettering', msg, destAddr) send_dead = True if send_dead: try: (self._parentQ or self._adminQ).put( deadQValue(for_local_addr if self._parentQ else None), True, timePeriodSeconds(MAX_QUEUE_TRANSMIT_PERIOD)) except Q.Full: thesplog('Unable to send deadmsg %s to %s or admin; discarding', msg, destAddr) continue # Not sure how to route this message yet. It # could be a heretofore silent child of one of our # children, it could be our parent (whose address # we don't know), or it could be elsewhere in the # tree. # # Try sending it to the parent first. If the # parent can't determine the routing, it will be # sent back down (relayAddr will be None in that # case) and it must be sprayed out to all children # in case the target lives somewhere beneath us. # Note that _parentQ will be None for top-level # actors, which send up to the Admin instead. # # As a special case, the external system is the # parent of the admin, but the admin is the # penultimate parent of all others, so this code # must keep the admin and the parent from playing # ping-pong with the message. But... the message # might be directed to the external system, which # is the parent of the Admin, so we need to check # with it first. # parentQ == None but adminQ good --> external # parentQ and adminQ and myAddress == adminAddr --> Admin # parentQ and adminQ and myAddress != adminADdr --> other Actor if relayAddr: # Send message up to the parent to see if the # parent knows how to forward it try: (self._parentQ or self._adminQ).put( putQValue(for_local_addr if self._parentQ else None), True, timePeriodSeconds(MAX_QUEUE_TRANSMIT_PERIOD)) except Q.Full: thesplog('Unable to send dead msg %s to %s or admin; discarding', msg, destAddr) else: # Sent by parent or we are an external, so this # may be some grandchild not currently known. # Do the worst case and just send this message # to ALL immediate children, hoping it will # get there via some path. for A,AQ in self._queues.items(): if A not in [self._adminAddr, str(self._adminAddr)]: # None means sent by Parent, so don't # send BACK to parent if unknown try: AQ.put(putQValue(None), True, timePeriodSeconds(MAX_QUEUE_TRANSMIT_PERIOD)) except Q.Full: pass return None def interrupt_run(self, signal_shutdown=False, check_children=False): self._shutdownSignalled |= signal_shutdown self._checkChildren |= check_children # Do not put anything on the Queue if running in the context # of a signal handler, because Queues are not signal-context # safe. Instead, those will just have to depend on the short # maximum Queue get wait time. if not signal_shutdown and not check_children: self._myInputQ.put_nowait('BuMP')
class AdminCore(systemCommonBase): def __init__(self, transport, address, capabilities, logdefs, concurrency_context): thesplog('++++ Starting Admin from %s', sys.modules['thespian'].__file__, level=logging.DEBUG) super(AdminCore, self).__init__(address, transport) self.init_replicator(transport, concurrency_context) self.capabilities = capabilities self.logdefs = logdefs self._pendingChildren = {} # Use: childLocalAddr instance # : PendingActorEnvelope # Things that help us look like an Actor, even though we're not self._sourceHash = None thesplog('++++ Admin started @ %s / gen %s', self.transport.myAddress, str(ThespianGeneration), level=logging.INFO, primary=True) logging.info('++++ Actor System gen %s started, admin @ %s', str(ThespianGeneration), self.transport.myAddress) logging.debug('Thespian source: %s', sys.modules['thespian'].__file__) self._nannying = AssocList() # child actorAddress -> parent Address self._deadLetterHandler = None self._sources = {} # Index is sourcehash, value PendingSource or ValidSource self._sourceAuthority = None self._sourceNotifications = [] # array of notification addresses def _activate(self): """Called when the full ActorSystem initialization is completed. This should then activate any functionality that needed to wait for completion of initialization. """ pass def run(self): try: while True: r = self.transport.run(self.handleIncoming, None) if isinstance(r, Thespian__UpdateWork): # tickle the transmit queues self._send_intent( TransmitIntent(self.myAddress, r)) continue # Expects that on completion of self.transport.run # that the Actor is done processing and that it has # been shutdown gracefully. break except Exception: import traceback thesplog('ActorAdmin uncaught exception: %s', traceback.format_exc(), level=logging.ERROR, exc_info=True) thesplog('Admin time to die', level=logging.DEBUG) def handleIncoming(self, envelope): self._sCBStats.inc('Admin Message Received.Total') handled, result = self._handleReplicatorMessages(envelope) if handled: return result if isinstance(envelope.message, (ActorSystemMessage, logging.LogRecord)): thesplog('Admin of %s', envelope.identify(), level=logging.DEBUG) return getattr(self, 'h_' + envelope.message.__class__.__name__, self.unrecognized)(envelope) # else discard random non-admin messages self._sCBStats.inc('Admin Message Received.Ignored') thesplog('ADMIN DISCARD %s', envelope.identify(), level=logging.WARNING) return True def unrecognized(self, envelope): self._sCBStats.inc('Admin Message Received.Discarded') thesplog("Admin got incoming %s from %s;" " discarded because I don't know how to handle it!", envelope.message, envelope.sender, level=logging.WARNING, primary=True) return True def isShuttingDown(self): return hasattr(self, '_exiting') def h_QueryExists(self, envelope): self._sCBStats.inc('Admin Message Received.Type.QueryExists') self._send_intent( TransmitIntent( envelope.sender, QueryAck( self.capabilities.get('Thespian ActorSystem Name', 'misc Actor System'), self.capabilities.get('Thespian ActorSystem Version', 'unknown Version'), self.isShuttingDown()))) return True def getStatus(self): resp = Thespian_SystemStatus(self.myAddress, capabilities=self.capabilities, inShutdown=self.isShuttingDown()) resp.setDeadLetterHandler(self._deadLetterHandler) self._updateStatusResponse(resp) resp.setLoadedSources(self._sources) resp.sourceAuthority = self._sourceAuthority return resp def h_Thespian_StatusReq(self, envelope): self._sCBStats.inc('Admin Message Received.Type.StatusReq') self._send_intent(TransmitIntent(envelope.sender, self.getStatus())) return True def thesplogStatus(self): "Write status to thesplog" from io import StringIO sd = StringIO() from thespian.system.messages.status import formatStatus try: sd.write('') SSD = lambda v: v except TypeError: class SSD(object): def __init__(self, sd): self.sd = sd def write(self, str_arg): self.sd.write(str_arg.decode('utf-8')) formatStatus(self.getStatus(), tofd=SSD(sd)) thesplog('STATUS: %s', sd.getvalue()) def h_SetLogging(self, envelope): return self.setLoggingControls(envelope) def h_SystemShutdown(self, envelope): self._exiting = envelope.sender thesplog('---- shutdown initiated by %s', envelope.sender, level=logging.DEBUG) # Send failure notices and clear out any pending children. If # any pending child ready notifications are received after # this, they will automatically be sent an ActorExitRequest. for each in self._pendingChildren: pendingReq = self._pendingChildren[each] self._send_intent( TransmitIntent( pendingReq.sender, PendingActorResponse( pendingReq.message.forActor, pendingReq.message.instanceNum, pendingReq.message.globalName, errorCode=PendingActorResponse.ERROR_ActorSystem_Shutting_Down))) self._pendingChildren = [] if not self.childAddresses: # no children? self._sayGoodbye() self.transport.abort_run(drain=True) return True # Now shutdown any direct children self._killLocalActors() # Callback will shutdown the Admin Once the children confirm # their exits. return True def _remove_expired_sources(self): rmvlist = [] for each in self._sources: if not self._sources[each].source_valid and \ self._sources[each].load_expires.expired(): rmvlist.append(each) for each in rmvlist: self._cancel_pending_actors(self._sources[each].pending_actors) del self._sources[each] def _cancel_pending_actors(self, pending_envelopes, error_code=PendingActorResponse.ERROR_Invalid_SourceHash): for each in pending_envelopes: self._sendPendingActorResponse(each, None, errorCode = error_code) def _killLocalActors(self): for each in self.childAddresses: self._send_intent( TransmitIntent(each, ActorExitRequest(recursive=True), onError=lambda r, m, a=each: self._handleChildExited(a))) def _sayGoodbye(self): self._cleanupAdmin() self._send_intent(TransmitIntent(self._exiting, SystemShutdownCompleted())) thesplog('---- shutdown completed', level=logging.INFO) logging.info('---- Actor System shutdown') self.shutdown_completed = True def h_ChildActorExited(self, envelope): return self._handleChildExited(envelope.message.childAddress) def _handleChildExited(self, childAddress): self._sourceNotifications = list(filter(lambda a: a != childAddress, self._sourceNotifications)) parentAddr = self._nannying.find(childAddress) if parentAddr: self._nannying.rmv(childAddress) # Let original requesting Actor (that *thinks* it's the # parent) know about this child exit as well. self._send_intent(TransmitIntent(parentAddr, ChildActorExited(childAddress))) return super(AdminCore, self)._handleChildExited(childAddress) def h_PendingActor(self, envelope): """Admin is creating an Actor. This covers one of the following cases: 1. Creating for external requester via ActorSystem. envelope.message.forActor will be None The Admin is the "parent" for all externally-created Actors. 2. Creating for another Actor when direct creation fails. Usually means that the current ActorSyste cannot meet the new Actor's requirements and the Admin should find another convention member that can create the new Actor. 3. GlobalName Actor. The Admin is the "parent" for all GlobalName Actors. """ self._sCBStats.inc('Admin Message Received.Type.Pending Actor Request') if self.isShuttingDown(): self._sendPendingActorResponse( envelope, None, errorCode=PendingActorResponse.ERROR_ActorSystem_Shutting_Down) return True sourceHash = envelope.message.sourceHash if sourceHash: self._remove_expired_sources() if sourceHash not in self._sources: self._sendPendingActorResponse( envelope, None, errorCode = PendingActorResponse.ERROR_Invalid_SourceHash) return True if not self._sources[sourceHash].source_valid: self._sources[sourceHash].pending_actors.append(envelope) return True # Note, both Admin and remote requester will have a local # child address for the child (with a different instanceNumber # each). childAddr = self._addrManager.createLocalAddress() childInstance = childAddr.addressDetails.addressInstanceNum try: self._startChildActor( childAddr, envelope.message.actorClassName, parentAddr=self.myAddress, # Admin is surrogate parent notifyAddr=self.myAddress, childRequirements=envelope.message.targetActorReq, sourceHash=sourceHash, sourceToLoad=(self._sources[sourceHash] if sourceHash else None)) except NoCompatibleSystemForActor: self._sendPendingActorResponse( envelope, None, errorCode=PendingActorResponse.ERROR_No_Compatible_ActorSystem) self._retryPendingChildOperations(childInstance, None) return True self._pendingChildren[childInstance] = envelope # transport will contrive to call _pendingActorReady when the # child is initialized and connected to this parent. return True def _sendPendingActorResponse(self, requestEnvelope, actualAddress, errorCode=None, errorStr=None): # actualAddress is None for failure if actualAddress is None and errorCode is None: raise ValueError('Must specify either actualAddress or errorCode') self._send_intent( TransmitIntent( requestEnvelope.message.forActor or requestEnvelope.sender, PendingActorResponse(requestEnvelope.message.forActor, requestEnvelope.message.instanceNum, requestEnvelope.message.globalName, errorCode=errorCode, errorStr=errorStr, actualAddress=actualAddress))) def _pendingActorReady(self, childInstance, actualAddress): if childInstance not in self._pendingChildren: thesplog('Pending actor is ready at %s for UNKNOWN %s' '; sending child a shutdown', actualAddress, childInstance, level=logging.WARNING) self._send_intent( TransmitIntent(actualAddress, ActorExitRequest(recursive=True))) return requestEnvelope = self._pendingChildren[childInstance] del self._pendingChildren[childInstance] if requestEnvelope.message.globalName or \ not requestEnvelope.message.forActor: # The Admin is the responsible Parent for these children self._registerChild(actualAddress) else: # Anything the Admin was requested to create is a adoptive # child and should be killed when the Admin exits. self._registerChild(actualAddress) if requestEnvelope.message.forActor: # Proxy-parenting; remember the real parent self._nannying.add(actualAddress, requestEnvelope.message.forActor) self._addrManager.associateUseableAddress(self.myAddress, childInstance, actualAddress) # n.b. childInstance is for this Admin, but caller's # childInstance is in original request self._sendPendingActorResponse(requestEnvelope, actualAddress) self._retryPendingChildOperations(childInstance, actualAddress) def h_HandleDeadLetters(self, envelope): # handlerAddr, enableHandler self._sCBStats.inc('Admin Message Received.Type.Set Dead Letter Handler') if envelope.message.enableHandler: self._deadLetterHandler = envelope.message.handlerAddr else: if self._deadLetterHandler == envelope.message.handlerAddr: self._deadLetterHandler = None return True def h_DeadEnvelope(self, envelope): self._sCBStats.inc('Admin Message Received.Type.Dead Letter') if self._deadLetterHandler: self._send_intent( TransmitIntent(self._deadLetterHandler, envelope.message)) return True def h_RegisterSourceAuthority(self, envelope): self._sourceAuthority = envelope.message.authorityAddress def h_NotifyOnSourceAvailability(self, envelope): address = envelope.message.notificationAddress enable = envelope.message.enable all_except = [A for A in self._sourceNotifications if A != address] if enable: self._sourceNotifications = all_except + [address] for each in self._sources: if self._sources[each].source_valid: self._send_intent( TransmitIntent(address, LoadedSource(self._sources[each].srcHash, self._sources[each].srcInfo))) else: self._sourceNotifications = all_except def h_ValidateSource(self, envelope): self._remove_expired_sources() sourceHash = envelope.message.sourceHash if not envelope.message.sourceData: self.unloadActorSource(sourceHash) logging.getLogger('Thespian')\ .info('Source hash %s unloaded', sourceHash) return if sourceHash in self._sources and \ self._sources[sourceHash].source_valid: logging.getLogger('Thespian')\ .info('Source hash %s (%s) already loaded', sourceHash, self._sources[sourceHash].srcInfo if isinstance(self._sources[sourceHash], ValidSource) else '<pending>') return if self._sourceAuthority: self._sources[sourceHash] = PendingSource(sourceHash, envelope.message.sourceData) self._send_intent(TransmitIntent(self._sourceAuthority, envelope.message)) return # Any attempt to load sources is ignored if there is no active # Source Authority. This is a security measure to protect the # un-protected. logging.getLogger('Thespian').warning( 'No source authority to validate source hash %s', sourceHash) def h_ValidatedSource(self, envelope): self._remove_expired_sources() if envelope.sender != self._sourceAuthority: logging.getLogger('Thespian').warning( 'Ignoring validated source from %s: not the source authority at %s', envelope.sender, self._sourceAuthority) return source_hash = envelope.message.sourceHash if envelope.message.sourceZip: self._loadValidatedActorSource( source_hash, envelope.message.sourceZip, getattr(envelope.message, 'sourceInfo', None)) logging.getLogger('Thespian').info( 'Source hash %s (%s) validated by Source Authority' '; now available.', source_hash, getattr(envelope.message, 'sourceInfo', '-')) else: # Source Authority actively rejected this source, so # actively unloaded it. Alternatively the Source # Authority can do nothing and this load attempt will # timeout. self._cancel_pending_actors( self._sources[source_hash].pending_actors) logging.getLogger('Thespian').warning( 'Source hash %s (%s) REJECTED by Source Authority', source_hash, getattr(envelope.message, 'sourceInfo', '-')) del self._sources[source_hash] def _loadValidatedActorSource(self, sourceHash, sourceZip, sourceInfo): # Validate the source file; this doesn't actually utilize the # sourceZip, but it ensures that the sourceZip isn't garbage # before registering it as active source. if sourceHash not in self._sources: logging.getLogger('Thespian').warning( 'Provided validated source with no or expired request' ', hash %s; ignoring.', sourceHash) return try: f = SourceHashFinder(sourceHash, lambda v: v, sourceZip) namelist = f.getZipNames() logging.getLogger('Thespian').info( 'Validated source hash %s - %s, %s modules (%s)', sourceHash, sourceInfo, len(namelist), ', '.join(namelist if len(namelist) < 10 else namelist[:9] + ['...'])) except Exception as ex: logging.getLogger('Thespian')\ .error('Validated source (hash %s) is corrupted: %s', sourceHash, ex) return if self._sources[sourceHash].source_valid: # If a duplicate source load request is made while the # first is still being validated by the Source Authority, # another request will be sent to the Source Authority and # the latter response will be a duplicate here and can # simply be dropped. return # Store this registered source pending_actors = self._sources[sourceHash].pending_actors self._sources[sourceHash] = ValidSource(sourceHash, self._sources[sourceHash].orig_data, sourceZip, str(sourceInfo)) for each in pending_actors: self.h_PendingActor(each) msg = LoadedSource(self._sources[sourceHash].srcHash, self._sources[sourceHash].srcInfo) for each in self._sourceNotifications: self._send_intent(TransmitIntent(each, msg)) def unloadActorSource(self, sourceHash): if sourceHash in self._sources: msg = UnloadedSource(self._sources[sourceHash].srcHash, self._sources[sourceHash].srcInfo) for each in self._sourceNotifications: self._send_intent(TransmitIntent(each, msg)) del self._sources[sourceHash] for pnum, metapath in enumerate(sys.meta_path): if getattr(metapath, 'srcHash', None) == sourceHash: rmmods = [M for M in sys.modules if M and M.startswith(metapath.hashRoot())] for each in rmmods: del sys.modules[each] del sys.meta_path[pnum] break def h_CapabilityUpdate(self, envelope): if self._updSystemCapabilities(envelope.message.capabilityName, envelope.message.capabilityValue): self._capUpdateLocalActors() def _updSystemCapabilities(self, cName, cVal): updateLocals = False if cVal is not None: updateLocals = cName not in self.capabilities or \ self.capabilities[cName] != cVal self.capabilities[cName] = cVal else: if cName in self.capabilities: updateLocals = True del self.capabilities[cName] return updateLocals def _capUpdateLocalActors(self): newCaps = NewCapabilities(self.capabilities, self.myAddress) for each in self.childAddresses: self._send_intent(TransmitIntent(each, newCaps))
class LocalConventionState(object): def __init__(self, myAddress, capabilities, sCBStats, getConventionAddressFunc): self._myAddress = myAddress self._capabilities = capabilities self._sCBStats = sCBStats self._conventionMembers = AssocList() # key=Remote Admin Addr, value=ConventionMemberData self._conventionNotificationHandlers = [] self._getConventionAddr = getConventionAddressFunc self._conventionAddress = getConventionAddressFunc(capabilities) self._conventionRegistration = ExpirationTimer(CONVENTION_REREGISTRATION_PERIOD) self._has_been_activated = False self._invited = False # entered convention as a result of an explicit invite @property def myAddress(self): return self._myAddress @property def capabilities(self): return self._capabilities def updateStatusResponse(self, resp): resp.setConventionLeaderAddress(self.conventionLeaderAddr) resp.setConventionRegisterTime(self._conventionRegistration) for each in self._conventionMembers.values(): resp.addConventioneer(each.remoteAddress, each.registryValid) resp.setNotifyHandlers(self._conventionNotificationHandlers) def active_in_convention(self): # If this is the convention leader, it is automatically # active, otherwise this convention member should have a # convention leader and that leader should have an active # entry in the _conventionMembers table (indicating it has # updated this system with its information) return bool(self.conventionLeaderAddr and self._conventionMembers.find(self.conventionLeaderAddr)) @property def conventionLeaderAddr(self): return self._conventionAddress def isConventionLeader(self): # Might also be the leader if self.conventionLeaderAddr is None return self.conventionLeaderAddr == self.myAddress def capabilities_have_changed(self, new_capabilities): self._capabilities = new_capabilities return self.setup_convention() def setup_convention(self, activation=False): self._has_been_activated |= activation rmsgs = [] # If not specified in capabilities, don't override any invites # that may have been received. self._conventionAddress = self._getConventionAddr(self.capabilities) or \ self._conventionAddress leader_is_gone = (self._conventionMembers.find(self.conventionLeaderAddr) is None) \ if self.conventionLeaderAddr else True if not self.isConventionLeader() and self.conventionLeaderAddr: thesplog('Admin registering with Convention @ %s (%s)', self.conventionLeaderAddr, 'first time' if leader_is_gone else 're-registering', level=logging.INFO, primary=True) rmsgs.append( HysteresisSend(self.conventionLeaderAddr, ConventionRegister(self.myAddress, self.capabilities, leader_is_gone), onSuccess = self._setupConventionCBGood, onError = self._setupConventionCBError)) rmsgs.append(LogAggregator(self.conventionLeaderAddr)) self._conventionRegistration = ExpirationTimer(CONVENTION_REREGISTRATION_PERIOD) return rmsgs def _setupConventionCBGood(self, result, finishedIntent): self._sCBStats.inc('Admin Convention Registered') if hasattr(self, '_conventionLeaderMissCount'): delattr(self, '_conventionLeaderMissCount') def _setupConventionCBError(self, result, finishedIntent): self._sCBStats.inc('Admin Convention Registration Failed') if hasattr(self, '_conventionLeaderMissCount'): self._conventionLeaderMissCount += 1 else: self._conventionLeaderMissCount = 1 thesplog('Admin cannot register with convention @ %s (miss %d): %s', finishedIntent.targetAddr, self._conventionLeaderMissCount, result, level=logging.WARNING, primary=True) def got_convention_invite(self, sender): self._conventionAddress = sender self._invited = True return self.setup_convention() def got_convention_register(self, regmsg): # Called when remote convention member has sent a ConventionRegister message self._sCBStats.inc('Admin Handle Convention Registration') if self._invited and not self.conventionLeaderAddr: # Lost connection to an invitation-only convention. # Cannot join again until another invitation is received. return [] # Registrant may re-register if changing capabilities rmsgs = [] registrant = regmsg.adminAddress prereg = getattr(regmsg, 'preRegister', False) # getattr used; see definition existing = self._conventionMembers.find(registrant) thesplog('Got Convention %sregistration from %s (%s) (new? %s)', 'pre-' if prereg else '', registrant, 'first time' if regmsg.firstTime else 're-registering', not existing, level=logging.INFO) if registrant == self.myAddress: # Either remote failed getting an external address and is # using 127.0.0.1 or else this is a malicious attempt to # make us talk to ourselves. Ignore it. thesplog('Convention registration from %s is an invalid address; ignoring.', registrant, level=logging.WARNING) return rmsgs existingPreReg = ( # existing.preRegOnly # or existing.preRegistered existing.permanentEntry ) if existing else False notify = (not existing or existing.preRegOnly) and not prereg if regmsg.firstTime or not existing: if existing: existing = None notify = not prereg rmsgs.extend(self._remote_system_cleanup(registrant)) newmember = ConventionMemberData(registrant, regmsg.capabilities, prereg) if prereg or existingPreReg: newmember.preRegistered = PreRegistration() self._conventionMembers.add(registrant, newmember) else: existing.refresh(regmsg.capabilities, prereg or existingPreReg) if not prereg: existing.preRegOnly = False if not self.isConventionLeader(): self._conventionRegistration = ExpirationTimer(CONVENTION_REREGISTRATION_PERIOD) # Convention Members normally periodically initiate a # membership message, to which the leader confirms by # responding; if this was a pre-registration, that identifies # this system as the "leader" for that remote. Also, if the # remote sent this because it was a pre-registration leader, # it doesn't yet have all the member information so the member # should respond. #if self.isConventionLeader() or prereg or regmsg.firstTime: if prereg: rmsgs.append(HysteresisCancel(registrant)) rmsgs.append(TransmitIntent(registrant, ConventionInvite())) elif (self.isConventionLeader() or prereg or regmsg.firstTime or \ (existing and existing.permanentEntry)): # If we are the Convention Leader, this would be the point to # inform all other registrants of the new registrant. At # present, there is no reciprocity here, so just update the # new registrant with the leader's info. rmsgs.append( TransmitIntent(registrant, ConventionRegister(self.myAddress, self.capabilities))) if notify: rmsgs.extend(self._notifications_of( ActorSystemConventionUpdate(registrant, regmsg.capabilities, True))) return rmsgs def _notifications_of(self, msg): return [TransmitIntent(H, msg) for H in self._conventionNotificationHandlers] def add_notification_handler(self, addr): if addr not in self._conventionNotificationHandlers: self._conventionNotificationHandlers.append(addr) # Now update the registrant on the current state of all convention members return [TransmitIntent(addr, ActorSystemConventionUpdate(M.remoteAddress, M.remoteCapabilities, True)) for M in self._conventionMembers.values() if not M.preRegOnly] return [] def remove_notification_handler(self, addr): self._conventionNotificationHandlers = [ H for H in self._conventionNotificationHandlers if H != addr] def got_convention_deregister(self, deregmsg): self._sCBStats.inc('Admin Handle Convention De-registration') remoteAdmin = deregmsg.adminAddress if remoteAdmin == self.myAddress: # Either remote failed getting an external address and is # using 127.0.0.1 or else this is a malicious attempt to # make us talk to ourselves. Ignore it. thesplog('Convention deregistration from %s is an invalid address; ignoring.', remoteAdmin, level=logging.WARNING) rmsgs = [] if getattr(deregmsg, 'preRegistered', False): # see definition for getattr use existing = self._conventionMembers.find(remoteAdmin) if existing: existing.preRegistered = None rmsgs.append(TransmitIntent(remoteAdmin, ConventionDeRegister(self.myAddress))) return rmsgs + self._remote_system_cleanup(remoteAdmin) def got_system_shutdown(self): return self.exit_convention() def exit_convention(self): self.invited = False gen_ops = lambda addr: [HysteresisCancel(addr), TransmitIntent(addr, ConventionDeRegister(self.myAddress)), ] terminate = lambda a: [ self._remote_system_cleanup(a), gen_ops(a) ][-1] if self.conventionLeaderAddr and \ self.conventionLeaderAddr != self.myAddress: thesplog('Admin de-registering with Convention @ %s', str(self.conventionLeaderAddr), level=logging.INFO, primary=True) # Cache convention leader address because it might get reset by terminate() claddr = self.conventionLeaderAddr terminate(self.conventionLeaderAddr) return gen_ops(claddr) return join(fmap(terminate, [M.remoteAddress for M in self._conventionMembers.values() if M.remoteAddress != self.myAddress])) def check_convention(self): rmsgs = [] if self._has_been_activated: rmsgs = foldl(lambda x, y: x + y, [self._check_preregistered_ping(member) for member in self._conventionMembers.values()], self._convention_leader_checks() if self.isConventionLeader() or not self.conventionLeaderAddr else self._convention_member_checks()) if self._conventionRegistration.expired(): self._conventionRegistration = ExpirationTimer(CONVENTION_REREGISTRATION_PERIOD) return rmsgs def _convention_leader_checks(self): return foldl(lambda x, y: x + y, [self._missed_checkin_remote_cleanup(R) for R in [ member for member in self._conventionMembers.values() if member.registryValid.expired() ]], []) def _missed_checkin_remote_cleanup(self, remote_member): thesplog('%s missed %d checkins (%s); assuming it has died', str(remote_member), CONVENTION_REGISTRATION_MISS_MAX, str(remote_member.registryValid), level=logging.WARNING, primary=True) return self._remote_system_cleanup(remote_member.remoteAddress) def _convention_member_checks(self): rmsgs = [] # Re-register with the Convention if it's time if self.conventionLeaderAddr and self._conventionRegistration.expired(): if getattr(self, '_conventionLeaderMissCount', 0) >= \ CONVENTION_REGISTRATION_MISS_MAX: thesplog('Admin convention registration lost @ %s (miss %d)', self.conventionLeaderAddr, self._conventionLeaderMissCount, level=logging.WARNING, primary=True) rmsgs.extend(self._remote_system_cleanup(self.conventionLeaderAddr)) self._conventionLeaderMissCount = 0 else: rmsgs.extend(self.setup_convention()) return rmsgs def _check_preregistered_ping(self, member): if member.preRegistered and \ member.preRegistered.pingValid.expired() and \ not member.preRegistered.pingPending: member.preRegistered.pingPending = True # If remote misses a checkin, re-extend the # invitation. This also helps re-initiate a socket # connection if a TxOnly socket has been lost. member.preRegistered.pingValid = ExpirationTimer( convention_reinvite_adjustment( CONVENTION_RESTART_PERIOD if member.registryValid.expired() else CONVENTION_REREGISTRATION_PERIOD)) return [HysteresisSend(member.remoteAddress, ConventionInvite(), onSuccess = self._preRegQueryNotPending, onError = self._preRegQueryNotPending)] return [] def _preRegQueryNotPending(self, result, finishedIntent): remoteAddr = finishedIntent.targetAddr member = self._conventionMembers.find(remoteAddr) if member and member.preRegistered: member.preRegistered.pingPending = False def _remote_system_cleanup(self, registrant): """Called when a RemoteActorSystem has exited and all associated Actors should be marked as exited and the ActorSystem removed from Convention membership. This is also called on a First Time connection from the remote to discard any previous connection information. """ thesplog('Convention cleanup or deregistration for %s (known? %s)', registrant, bool(self._conventionMembers.find(registrant)), level=logging.INFO) rmsgs = [LostRemote(registrant)] cmr = self._conventionMembers.find(registrant) if not cmr or cmr.preRegOnly: return [] # Send exited notification to conventionNotificationHandler (if any) for each in self._conventionNotificationHandlers: rmsgs.append( TransmitIntent(each, ActorSystemConventionUpdate(cmr.remoteAddress, cmr.remoteCapabilities, False))) # errors ignored # If the remote ActorSystem shutdown gracefully (i.e. sent # a Convention Deregistration) then it should not be # necessary to shutdown remote Actors (or notify of their # shutdown) because the remote ActorSystem should already # have caused this to occur. However, it won't hurt, and # it's necessary if the remote ActorSystem did not exit # gracefully. for lpa, raa in cmr.hasRemoteActors: # ignore errors: rmsgs.append(TransmitIntent(lpa, ChildActorExited(raa))) # n.b. at present, this means that the parent might # get duplicate notifications of ChildActorExited; it # is expected that Actors can handle this. # Remove remote system from conventionMembers if not cmr.preRegistered: if registrant == self.conventionLeaderAddr and self._invited: self._conventionAddress = None # Don't clear invited: once invited, that # perpetually indicates this should be only a # member and never a leader. self._conventionMembers.rmv(registrant) else: # This conventionMember needs to stay because the # current system needs to continue issuing # registration pings. By setting the registryValid # expiration to forever, this member won't re-time-out # and will therefore be otherwise ignored... until it # registers again at which point the membership will # be updated with new settings. cmr.registryValid = ExpirationTimer(None) cmr.preRegOnly = True return rmsgs + [HysteresisCancel(registrant)] def sentByRemoteAdmin(self, envelope): for each in self._conventionMembers.values(): if envelope.sender == each.remoteAddress: return True return False def convention_inattention_delay(self): return self._conventionRegistration or \ ExpirationTimer(CONVENTION_REREGISTRATION_PERIOD if self.active_in_convention() or self.isConventionLeader() else CONVENTION_RESTART_PERIOD) def forward_pending_to_remote_system(self, childClass, envelope, sourceHash, acceptsCaps): alreadyTried = getattr(envelope.message, 'alreadyTried', []) remoteCandidates = [ K for K in self._conventionMembers.values() if not K.registryValid.expired() and K.remoteAddress != envelope.sender # source Admin and K.remoteAddress not in alreadyTried and acceptsCaps(K.remoteCapabilities)] if not remoteCandidates: if self.isConventionLeader() or not self.conventionLeaderAddr: raise NoCompatibleSystemForActor( childClass, 'No known ActorSystems can handle a %s for %s', childClass, envelope.message.forActor) # Let the Convention Leader try to find an appropriate ActorSystem bestC = self.conventionLeaderAddr else: # distribute equally amongst candidates C = [(K.remoteAddress, len(K.hasRemoteActors)) for K in remoteCandidates] bestC = foldl(lambda best,possible: best if best[1] <= possible[1] else possible, C)[0] thesplog('Requesting creation of %s%s on remote admin %s', envelope.message.actorClassName, ' (%s)'%sourceHash if sourceHash else '', bestC) if bestC not in alreadyTried: # Don't send request to this remote again, it has already # been tried. This would also be indicated by that system # performing the add of self.myAddress as below, but if # there is disagreement between the local and remote # addresses, this addition will prevent continual # bounceback. alreadyTried.append(bestC) if self.myAddress not in alreadyTried: # Don't send request back to this actor system: it cannot # handle it alreadyTried.append(self.myAddress) envelope.message.alreadyTried = alreadyTried return [TransmitIntent(bestC, envelope.message)] def send_to_all_members(self, message, exception_list=None): return [HysteresisSend(M.remoteAddress, message) for M in self._conventionMembers.values() if M.remoteAddress not in (exception_list or [])]
class MultiprocessQueueTCore_Common(object): def __init__(self, myQueue, parentQ, adminQ, adminAddr): self._myInputQ = myQueue self._parentQ = parentQ self._adminQ = adminQ self._adminAddr = adminAddr # _queues is a map of direct child ActorAddresses to Queue instance. Note # that there will be multiple keys mapping to the same Queue # instance because routing is only either to the Parent or to # an immediate Child. self._queues = AssocList() # addr -> queue # _fwdvia represents routing for other than immediate parent # or child (there may be multiple target addresses mapping to # the same forward address. self._fwdvia = AssocList() # targetAddress -> fwdViaAddress self._deadaddrs = [] # Signals can set these to true; they should be checked and # reset by the main processing loop. There is a small window # where they could be missed because signals are not queued, # but this should handle the majority of situations. Note # that the Queue object is NOT signal-safe, so don't try to # queue signals that way. self._checkChildren = False self._shutdownSignalled = False def mainLocalInputQueueEndpoint(self): return self._myInputQ def adminQueueEndpoint(self): return self._adminQ @property def adminAddr(self): return self._adminAddr def protectedFileNumList(self): return foldl(lambda a, b: a + [b._reader.fileno( ), b._writer.fileno()], [self._myInputQ, self._parentQ, self._adminQ] + list(self._queues.values()), []) def childResetFileNumList(self): return foldl(lambda a, b: a + [b._reader.fileno(), b._writer.fileno()], [self._parentQ] + list(self._queues.values()), []) def add_endpoint(self, child_addr, child_queue): self._queues.add(child_addr, child_queue) def set_address_to_dead(self, child_addr): self._queues.rmv(child_addr) self._fwdvia.rmv(child_addr) self._fwdvia.rmv_value(child_addr) self._deadaddrs.append(child_addr) def abort_core_run(self): self._aborting_run = Thespian__Run_Terminated() def core_common_transmit(self, transmit_intent, from_addr): try: if self.isMyAddress(transmit_intent.targetAddr): if transmit_intent.message: self._myInputQ.put( (from_addr, transmit_intent.serMsg), True, timePeriodSeconds(transmit_intent.delay())) else: tgtQ = self._queues.find(transmit_intent.targetAddr) if tgtQ: tgtQ.put((from_addr, transmit_intent.serMsg), True, timePeriodSeconds(transmit_intent.delay())) else: # None means sent by parent, so don't send BACK to parent if unknown topOrFromBelow = from_addr if self._parentQ else None (self._parentQ or self._adminQ).put( (topOrFromBelow, transmit_intent.serMsg), True, timePeriodSeconds(transmit_intent.delay())) transmit_intent.tx_done(SendStatus.Sent) return except Q.Full: pass transmit_intent.tx_done( SendStatus.DeadTarget if not isinstance(transmit_intent._message, ( ChildActorExited, ActorExitRequest)) else SendStatus.Failed) def core_common_receive(self, incoming_handler, local_routing_addr, run_time_f): """Core scheduling method; called by the current Actor process when idle to await new messages (or to do background processing). """ if incoming_handler == TransmitOnly or \ isinstance(incoming_handler, TransmitOnly): # transmits are not queued/multistage in this transport, no waiting return local_routing_addr, 0 self._aborting_run = None while self._aborting_run is None: ct = currentTime() if run_time_f().view(ct).expired(): break try: # Unfortunately, the Queue object is not signal-safe, # so a frequent wakeup is needed to check # _checkChildren and _shutdownSignalled. rcvd = self._myInputQ.get( True, min( run_time_f().view(ct).remainingSeconds() or QUEUE_CHECK_PERIOD, QUEUE_CHECK_PERIOD)) except Q.Empty: if not self._checkChildren and not self._shutdownSignalled: # Probably a timeout, but let the while loop decide for sure continue rcvd = 'BuMP' if rcvd == 'BuMP': relayAddr = sendAddr = destAddr = local_routing_addr if self._checkChildren: self._checkChildren = False msg = ChildMayHaveDied() elif self._shutdownSignalled: self._shutdownSignalled = False msg = ActorExitRequest() else: return local_routing_addr, Thespian__UpdateWork() else: relayAddr, (sendAddr, destAddr, msg) = rcvd if not self._queues.find(sendAddr): # We don't directly know about this sender, so # remember what path this arrived on to know where to # direct future messages for this sender. if relayAddr and self._queues.find(relayAddr) and \ not self._fwdvia.find(sendAddr): # relayAddr might be None if it's our parent, which is OK because # the default message forwarding is to the parent. If it's not # none, it should be in self._queues though! self._fwdvia.add(sendAddr, relayAddr) if hasattr(self, '_addressMgr'): destAddr, msg = self._addressMgr.prepMessageSend(destAddr, msg) if destAddr is None: thesplog('Unexpected target inaccessibility for %s', msg, level=logging.WARNING) raise CannotPickleAddress(destAddr) if msg is SendStatus.DeadTarget: thesplog( 'Faking message "sent" because target is dead and recursion avoided.' ) continue if self.isMyAddress(destAddr): if isinstance(incoming_handler, ReturnTargetAddressWithEnvelope): return destAddr, ReceiveEnvelope(sendAddr, msg) if incoming_handler is None: return destAddr, ReceiveEnvelope(sendAddr, msg) r = Thespian__Run_HandlerResult( incoming_handler(ReceiveEnvelope(sendAddr, msg))) if not r: # handler returned False-ish, indicating run() should exit return destAddr, r else: # Note: the following code has implicit knowledge of serialize() and xmit putQValue = lambda relayer: (relayer, (sendAddr, destAddr, msg)) deadQValue = lambda relayer: (relayer, (sendAddr, self._adminAddr, DeadEnvelope(destAddr, msg))) # Must forward this packet via a known forwarder or our parent. send_dead = False tgtQ = self._queues.find(destAddr) if tgtQ: sendArgs = putQValue(local_routing_addr), True if not tgtQ: tgtA = self._fwdvia.find(destAddr) if tgtA: tgtQ = self._queues.find(tgtA) sendArgs = putQValue(None), else: for each in self._deadaddrs: if destAddr == each: send_dead = True if tgtQ: try: tgtQ.put(*sendArgs, timeout=timePeriodSeconds( MAX_QUEUE_TRANSMIT_PERIOD)) continue except Q.Full: thesplog( 'Unable to send msg %s to dest %s; dead lettering', msg, destAddr) send_dead = True if send_dead: try: (self._parentQ or self._adminQ).put( deadQValue( local_routing_addr if self._parentQ else None), True, timePeriodSeconds(MAX_QUEUE_TRANSMIT_PERIOD)) except Q.Full: thesplog( 'Unable to send deadmsg %s to %s or admin; discarding', msg, destAddr) continue # Not sure how to route this message yet. It # could be a heretofore silent child of one of our # children, it could be our parent (whose address # we don't know), or it could be elsewhere in the # tree. # # Try sending it to the parent first. If the # parent can't determine the routing, it will be # sent back down (relayAddr will be None in that # case) and it must be sprayed out to all children # in case the target lives somewhere beneath us. # Note that _parentQ will be None for top-level # actors, which send up to the Admin instead. # # As a special case, the external system is the # parent of the admin, but the admin is the # penultimate parent of all others, so this code # must keep the admin and the parent from playing # ping-pong with the message. But... the message # might be directed to the external system, which # is the parent of the Admin, so we need to check # with it first. # parentQ == None but adminQ good --> external # parentQ and adminQ and myAddress == adminAddr --> Admin # parentQ and adminQ and myAddress != adminADdr --> other Actor if relayAddr: # Send message up to the parent to see if the # parent knows how to forward it try: (self._parentQ or self._adminQ).put( putQValue( local_routing_addr if self._parentQ else None), True, timePeriodSeconds(MAX_QUEUE_TRANSMIT_PERIOD)) except Q.Full: thesplog( 'Unable to send dead msg %s to %s or admin; discarding', msg, destAddr) else: # Sent by parent or we are an external, so this # may be some grandchild not currently known. # Do the worst case and just send this message # to ALL immediate children, hoping it will # get there via some path. for A, AQ in self._queues.items(): if A not in [self._adminAddr, str(self._adminAddr)]: # None means sent by Parent, so don't # send BACK to parent if unknown try: AQ.put( putQValue(None), True, timePeriodSeconds( MAX_QUEUE_TRANSMIT_PERIOD)) except Q.Full: pass if self._aborting_run is not None: return local_routing_addr, self._aborting_run return local_routing_addr, Thespian__Run_Expired() def interrupt_run(self, signal_shutdown=False, check_children=False): self._shutdownSignalled |= signal_shutdown self._checkChildren |= check_children # Do not put anything on the Queue if running in the context # of a signal handler, because Queues are not signal-context # safe. Instead, those will just have to depend on the short # maximum Queue get wait time. if not signal_shutdown and not check_children: try: self._myInputQ.put_nowait('BuMP') except Q.Full: # if the queue is full, it should be reading something # off soon which will accomplish the same interrupt # effect, so nothing else needs to be done here. pass
class PendingTransmits(object): def __init__(self, address_manager): self._addrmgr = address_manager # There are expected to be a low number of pending transmits # for a typical actor. At present time, this is an array of # intents. Note that an intent may fail due to a # CannotPickleAddress (which calls cannot_send_now() ), and # then later be passed here again to can_send_now() when the # address becomes known. self._atd = AssocList() # address -> ptl index self._ptl = [] def _intent_addresses(self, intent): yield intent.targetAddr xlated_addr = self._addrmgr.sendToAddress(intent.targetAddr) if xlated_addr: yield xlated_addr deadaddr = getattr(intent.message, 'deadAddress', None) if deadaddr: yield deadaddr xlated_addr = self._addrmgr.sendToAddress(deadaddr) if xlated_addr: yield xlated_addr def p_can_send_now(self, stats, intent): addrs = list(self._intent_addresses(intent)) for idx, addr in enumerate(addrs): ptloc = self._atd.find(addr) if ptloc is not None: for ii in range(idx): self._atd.add(addrs[ii], ptloc) break else: ptloc = len(self._ptl) self._ptl.append([]) for addr in addrs: self._atd.add(addr, ptloc) if not(self._ptl[ptloc]): self._ptl[ptloc] = [intent] return True self._ptl[ptloc].append(intent) return False def get_next(self, completed_intent): for addr in self._intent_addresses(completed_intent): ptloc = self._atd.find(addr) if ptloc is not None: break else: thesplog('No pending transmits for completed intent: %s', completed_intent, level=logging.ERROR) return None if not(self._ptl[ptloc]): thesplog('No pending entry for completed intent: %s', completed_intent, level=logging.ERROR) return None self._ptl[ptloc].pop(0) if self._ptl[ptloc]: return self._ptl[ptloc][0] # Trim here... return None def cannot_send_now(self, intent): return self.get_next(intent) def change_address_for_transmit(self, oldaddr, newaddr): oldidx = self._atd.find(oldaddr) if oldidx is None: # Have not scheduled any transmits for this (probably new) # child yet. return newidx = self._atd.find(newaddr) if newidx is None: self._atd.add(newaddr, oldidx) elif newidx != oldidx: if isinstance(oldaddr.addressDetails, ActorLocalAddress): # This can happen if sends are made to createActor # results with a globalName before the actual address # is known. Each createActor creates a local address, # but all those local addresses map back to the same # actual address. self._ptl[newidx].extend(self._ptl[oldidx]) self._atd.add(oldaddr, newidx) self._ptl[oldidx] = [] # should not be used anymore else: thesplog('Duplicate pending transmit indices' ': %s -> %s, %s -> %s', oldaddr, oldidx, newaddr, newidx, level=logging.ERROR) def update_status_response(self, stats, my_address): for group in self._ptl: for each in group: stats.addPendingMessage(my_address, each.targetAddr, each.message)
class LocalConventionState(object): def __init__(self, myAddress, capabilities, sCBStats, getConventionAddressFunc): self._myAddress = myAddress self._capabilities = capabilities self._sCBStats = sCBStats self._conventionMembers = AssocList( ) # key=Remote Admin Addr, value=ConventionMemberData self._conventionNotificationHandlers = [] self._getConventionAddr = getConventionAddressFunc self._conventionAddress = getConventionAddressFunc(capabilities) self._conventionRegistration = ExpirationTimer( CONVENTION_REREGISTRATION_PERIOD) self._has_been_activated = False self._invited = False # entered convention as a result of an explicit invite @property def myAddress(self): return self._myAddress @property def capabilities(self): return self._capabilities def updateStatusResponse(self, resp): resp.setConventionLeaderAddress(self.conventionLeaderAddr) resp.setConventionRegisterTime(self._conventionRegistration) for each in self._conventionMembers.values(): resp.addConventioneer(each.remoteAddress, each.registryValid) resp.setNotifyHandlers(self._conventionNotificationHandlers) def active_in_convention(self): # If this is the convention leader, it is automatically # active, otherwise this convention member should have a # convention leader and that leader should have an active # entry in the _conventionMembers table (indicating it has # updated this system with its information) return bool( self.conventionLeaderAddr and self._conventionMembers.find(self.conventionLeaderAddr)) @property def conventionLeaderAddr(self): return self._conventionAddress def isConventionLeader(self): # Might also be the leader if self.conventionLeaderAddr is None return self.conventionLeaderAddr == self.myAddress def capabilities_have_changed(self, new_capabilities): self._capabilities = new_capabilities return self.setup_convention() def setup_convention(self, activation=False): self._has_been_activated |= activation rmsgs = [] # If not specified in capabilities, don't override any invites # that may have been received. self._conventionAddress = self._getConventionAddr(self.capabilities) or \ self._conventionAddress leader_is_gone = (self._conventionMembers.find(self.conventionLeaderAddr) is None) \ if self.conventionLeaderAddr else True if not self.isConventionLeader() and self.conventionLeaderAddr: thesplog('Admin registering with Convention @ %s (%s)', self.conventionLeaderAddr, 'first time' if leader_is_gone else 're-registering', level=logging.INFO, primary=True) rmsgs.append( HysteresisSend(self.conventionLeaderAddr, ConventionRegister(self.myAddress, self.capabilities, leader_is_gone), onSuccess=self._setupConventionCBGood, onError=self._setupConventionCBError)) rmsgs.append(LogAggregator(self.conventionLeaderAddr)) self._conventionRegistration = ExpirationTimer( CONVENTION_REREGISTRATION_PERIOD) return rmsgs def _setupConventionCBGood(self, result, finishedIntent): self._sCBStats.inc('Admin Convention Registered') if hasattr(self, '_conventionLeaderMissCount'): delattr(self, '_conventionLeaderMissCount') def _setupConventionCBError(self, result, finishedIntent): self._sCBStats.inc('Admin Convention Registration Failed') if hasattr(self, '_conventionLeaderMissCount'): self._conventionLeaderMissCount += 1 else: self._conventionLeaderMissCount = 1 thesplog('Admin cannot register with convention @ %s (miss %d): %s', finishedIntent.targetAddr, self._conventionLeaderMissCount, result, level=logging.WARNING, primary=True) def got_convention_invite(self, sender): self._conventionAddress = sender self._invited = True return self.setup_convention() def got_convention_register(self, regmsg): # Called when remote convention member has sent a ConventionRegister message self._sCBStats.inc('Admin Handle Convention Registration') if self._invited and not self.conventionLeaderAddr: # Lost connection to an invitation-only convention. # Cannot join again until another invitation is received. return [] # Registrant may re-register if changing capabilities rmsgs = [] registrant = regmsg.adminAddress prereg = getattr(regmsg, 'preRegister', False) # getattr used; see definition existing = self._conventionMembers.find(registrant) thesplog('Got Convention %sregistration from %s (%s) (new? %s)', 'pre-' if prereg else '', registrant, 'first time' if regmsg.firstTime else 're-registering', not existing, level=logging.INFO) if registrant == self.myAddress: # Either remote failed getting an external address and is # using 127.0.0.1 or else this is a malicious attempt to # make us talk to ourselves. Ignore it. thesplog( 'Convention registration from %s is an invalid address; ignoring.', registrant, level=logging.WARNING) return rmsgs existingPreReg = ( # existing.preRegOnly # or existing.preRegistered existing.permanentEntry) if existing else False notify = (not existing or existing.preRegOnly) and not prereg if regmsg.firstTime or not existing: if existing: existing = None notify = not prereg rmsgs.extend(self._remote_system_cleanup(registrant)) newmember = ConventionMemberData(registrant, regmsg.capabilities, prereg) if prereg or existingPreReg: newmember.preRegistered = PreRegistration() self._conventionMembers.add(registrant, newmember) else: existing.refresh(regmsg.capabilities, prereg or existingPreReg) if not prereg: existing.preRegOnly = False if not self.isConventionLeader(): self._conventionRegistration = ExpirationTimer( CONVENTION_REREGISTRATION_PERIOD) # Convention Members normally periodically initiate a # membership message, to which the leader confirms by # responding; if this was a pre-registration, that identifies # this system as the "leader" for that remote. Also, if the # remote sent this because it was a pre-registration leader, # it doesn't yet have all the member information so the member # should respond. #if self.isConventionLeader() or prereg or regmsg.firstTime: if prereg: rmsgs.append(HysteresisCancel(registrant)) rmsgs.append(TransmitIntent(registrant, ConventionInvite())) elif (self.isConventionLeader() or prereg or regmsg.firstTime or \ (existing and existing.permanentEntry)): # If we are the Convention Leader, this would be the point to # inform all other registrants of the new registrant. At # present, there is no reciprocity here, so just update the # new registrant with the leader's info. rmsgs.append( TransmitIntent( registrant, ConventionRegister(self.myAddress, self.capabilities))) if notify: rmsgs.extend( self._notifications_of( ActorSystemConventionUpdate(registrant, regmsg.capabilities, True))) return rmsgs def _notifications_of(self, msg): return [ TransmitIntent(H, msg) for H in self._conventionNotificationHandlers ] def add_notification_handler(self, addr): if addr not in self._conventionNotificationHandlers: self._conventionNotificationHandlers.append(addr) # Now update the registrant on the current state of all convention members return [ TransmitIntent( addr, ActorSystemConventionUpdate(M.remoteAddress, M.remoteCapabilities, True)) for M in self._conventionMembers.values() if not M.preRegOnly ] return [] def remove_notification_handler(self, addr): self._conventionNotificationHandlers = [ H for H in self._conventionNotificationHandlers if H != addr ] def got_convention_deregister(self, deregmsg): self._sCBStats.inc('Admin Handle Convention De-registration') remoteAdmin = deregmsg.adminAddress if remoteAdmin == self.myAddress: # Either remote failed getting an external address and is # using 127.0.0.1 or else this is a malicious attempt to # make us talk to ourselves. Ignore it. thesplog( 'Convention deregistration from %s is an invalid address; ignoring.', remoteAdmin, level=logging.WARNING) rmsgs = [] if getattr(deregmsg, 'preRegistered', False): # see definition for getattr use existing = self._conventionMembers.find(remoteAdmin) if existing: existing.preRegistered = None rmsgs.append( TransmitIntent(remoteAdmin, ConventionDeRegister(self.myAddress))) return rmsgs + self._remote_system_cleanup(remoteAdmin) def got_system_shutdown(self): return self.exit_convention() def exit_convention(self): self.invited = False gen_ops = lambda addr: [ HysteresisCancel(addr), TransmitIntent(addr, ConventionDeRegister(self.myAddress)), ] terminate = lambda a: [self._remote_system_cleanup(a), gen_ops(a)][-1] if self.conventionLeaderAddr and \ self.conventionLeaderAddr != self.myAddress: thesplog('Admin de-registering with Convention @ %s', str(self.conventionLeaderAddr), level=logging.INFO, primary=True) # Cache convention leader address because it might get reset by terminate() claddr = self.conventionLeaderAddr terminate(self.conventionLeaderAddr) return gen_ops(claddr) return join( fmap(terminate, [ M.remoteAddress for M in self._conventionMembers.values() if M.remoteAddress != self.myAddress ])) def check_convention(self): ct = currentTime() rmsgs = [] if self._has_been_activated: rmsgs = foldl( lambda x, y: x + y, [ self._check_preregistered_ping(ct, member) for member in self._conventionMembers.values() ], self._convention_leader_checks(ct) if self.isConventionLeader() or not self.conventionLeaderAddr else self._convention_member_checks(ct)) if self._conventionRegistration.view(ct).expired(): self._conventionRegistration = ExpirationTimer( CONVENTION_REREGISTRATION_PERIOD) return rmsgs def _convention_leader_checks(self, ct): return foldl(lambda x, y: x + y, [ self._missed_checkin_remote_cleanup(R) for R in [ member for member in self._conventionMembers.values() if member.registryValid.view(ct).expired() ] ], []) def _missed_checkin_remote_cleanup(self, remote_member): thesplog('%s missed %d checkins (%s); assuming it has died', str(remote_member), CONVENTION_REGISTRATION_MISS_MAX, str(remote_member.registryValid), level=logging.WARNING, primary=True) return self._remote_system_cleanup(remote_member.remoteAddress) def _convention_member_checks(self, ct): rmsgs = [] # Re-register with the Convention if it's time if self.conventionLeaderAddr and \ self._conventionRegistration.view(ct).expired(): if getattr(self, '_conventionLeaderMissCount', 0) >= \ CONVENTION_REGISTRATION_MISS_MAX: thesplog('Admin convention registration lost @ %s (miss %d)', self.conventionLeaderAddr, self._conventionLeaderMissCount, level=logging.WARNING, primary=True) rmsgs.extend( self._remote_system_cleanup(self.conventionLeaderAddr)) self._conventionLeaderMissCount = 0 else: rmsgs.extend(self.setup_convention()) return rmsgs def _check_preregistered_ping(self, ct, member): if member.preRegistered and \ member.preRegistered.pingValid.view(ct).expired() and \ not member.preRegistered.pingPending: member.preRegistered.pingPending = True # If remote misses a checkin, re-extend the # invitation. This also helps re-initiate a socket # connection if a TxOnly socket has been lost. member.preRegistered.pingValid = ExpirationTimer( convention_reinvite_adjustment( CONVENTION_RESTART_PERIOD if member.registryValid.view(ct). expired() else CONVENTION_REREGISTRATION_PERIOD)) return [ HysteresisSend(member.remoteAddress, ConventionInvite(), onSuccess=self._preRegQueryNotPending, onError=self._preRegQueryNotPending) ] return [] def _preRegQueryNotPending(self, result, finishedIntent): remoteAddr = finishedIntent.targetAddr member = self._conventionMembers.find(remoteAddr) if member and member.preRegistered: member.preRegistered.pingPending = False def _remote_system_cleanup(self, registrant): """Called when a RemoteActorSystem has exited and all associated Actors should be marked as exited and the ActorSystem removed from Convention membership. This is also called on a First Time connection from the remote to discard any previous connection information. """ thesplog('Convention cleanup or deregistration for %s (known? %s)', registrant, bool(self._conventionMembers.find(registrant)), level=logging.INFO) rmsgs = [LostRemote(registrant)] cmr = self._conventionMembers.find(registrant) if not cmr or cmr.preRegOnly: return [] # Send exited notification to conventionNotificationHandler (if any) for each in self._conventionNotificationHandlers: rmsgs.append( TransmitIntent( each, ActorSystemConventionUpdate(cmr.remoteAddress, cmr.remoteCapabilities, False))) # errors ignored # If the remote ActorSystem shutdown gracefully (i.e. sent # a Convention Deregistration) then it should not be # necessary to shutdown remote Actors (or notify of their # shutdown) because the remote ActorSystem should already # have caused this to occur. However, it won't hurt, and # it's necessary if the remote ActorSystem did not exit # gracefully. for lpa, raa in cmr.hasRemoteActors: # ignore errors: rmsgs.append(TransmitIntent(lpa, ChildActorExited(raa))) # n.b. at present, this means that the parent might # get duplicate notifications of ChildActorExited; it # is expected that Actors can handle this. # Remove remote system from conventionMembers if not cmr.preRegistered: if registrant == self.conventionLeaderAddr and self._invited: self._conventionAddress = None # Don't clear invited: once invited, that # perpetually indicates this should be only a # member and never a leader. self._conventionMembers.rmv(registrant) else: # This conventionMember needs to stay because the # current system needs to continue issuing # registration pings. By setting the registryValid # expiration to forever, this member won't re-time-out # and will therefore be otherwise ignored... until it # registers again at which point the membership will # be updated with new settings. cmr.registryValid = ExpirationTimer(None) cmr.preRegOnly = True return rmsgs + [HysteresisCancel(registrant)] def sentByRemoteAdmin(self, envelope): for each in self._conventionMembers.values(): if envelope.sender == each.remoteAddress: return True return False def convention_inattention_delay(self, current_time): return (self._conventionRegistration or ExpirationTimer( CONVENTION_REREGISTRATION_PERIOD if self.active_in_convention() or self.isConventionLeader() else CONVENTION_RESTART_PERIOD) ).view(current_time) def forward_pending_to_remote_system(self, childClass, envelope, sourceHash, acceptsCaps): alreadyTried = getattr(envelope.message, 'alreadyTried', []) ct = currentTime() remoteCandidates = [ K for K in self._conventionMembers.values() if not K.registryValid.view(ct).expired() and K.remoteAddress != envelope.sender # source Admin and K.remoteAddress not in alreadyTried and acceptsCaps(K.remoteCapabilities) ] if not remoteCandidates: if self.isConventionLeader() or not self.conventionLeaderAddr: raise NoCompatibleSystemForActor( childClass, 'No known ActorSystems can handle a %s for %s', childClass, envelope.message.forActor) # Let the Convention Leader try to find an appropriate ActorSystem bestC = self.conventionLeaderAddr else: # distribute equally amongst candidates C = [(K.remoteAddress, len(K.hasRemoteActors)) for K in remoteCandidates] bestC = foldl( lambda best, possible: best if best[1] <= possible[1] else possible, C)[0] thesplog('Requesting creation of %s%s on remote admin %s', envelope.message.actorClassName, ' (%s)' % sourceHash if sourceHash else '', bestC) if bestC not in alreadyTried: # Don't send request to this remote again, it has already # been tried. This would also be indicated by that system # performing the add of self.myAddress as below, but if # there is disagreement between the local and remote # addresses, this addition will prevent continual # bounceback. alreadyTried.append(bestC) if self.myAddress not in alreadyTried: # Don't send request back to this actor system: it cannot # handle it alreadyTried.append(self.myAddress) envelope.message.alreadyTried = alreadyTried return [TransmitIntent(bestC, envelope.message)] def send_to_all_members(self, message, exception_list=None): return [ HysteresisSend(M.remoteAddress, message) for M in self._conventionMembers.values() if M.remoteAddress not in (exception_list or []) ]
class AdminCore(systemCommonBase): def __init__(self, transport, address, capabilities, logdefs, concurrency_context): thesplog('++++ Starting Admin from %s', sys.modules['thespian'].__file__, level=logging.DEBUG) super(AdminCore, self).__init__(address, transport) self.init_replicator(transport, concurrency_context) self.capabilities = capabilities self.logdefs = logdefs self._pendingChildren = { } # Use: childLocalAddr instance # : PendingActorEnvelope # Things that help us look like an Actor, even though we're not self._sourceHash = None thesplog('++++ Admin started @ %s / gen %s', self.transport.myAddress, str(ThespianGeneration), level=logging.INFO, primary=True) logging.info('++++ Actor System gen %s started, admin @ %s', str(ThespianGeneration), self.transport.myAddress) logging.debug('Thespian source: %s', sys.modules['thespian'].__file__) self._nannying = AssocList() # child actorAddress -> parent Address self._deadLetterHandler = None self._sources = { } # Index is sourcehash, value PendingSource or ValidSource self._sourceAuthority = None self._sourceNotifications = [] # array of notification addresses # The initialization of the Admin and its logger # occurs asynchronously, but since the Admin is using a known # address, there is nothing to prevent clients from initiating # requests to the Admin before it has had a chance to complete # the initialization; the _pre_init_msgs will hold those # requests until the initialization has completed. self._pre_init_msgs = [] def _activate(self): """Called when the full ActorSystem initialization is completed. This should then activate any functionality that needed to wait for completion of initialization. """ now_do = self._pre_init_msgs self._pre_init_msgs = [] for each in now_do: self.handleIncoming(each) def run(self): try: while True: r = self.transport.run(self.handleIncoming, None) if isinstance(r, Thespian__UpdateWork): # tickle the transmit queues self._send_intent(TransmitIntent(self.myAddress, r)) continue # Expects that on completion of self.transport.run # that the Actor is done processing and that it has # been shutdown gracefully. break except Exception: import traceback thesplog('ActorAdmin uncaught exception: %s', traceback.format_exc(), level=logging.ERROR, exc_info=True) thesplog('Admin time to die', level=logging.DEBUG) def handleIncoming(self, envelope): self._sCBStats.inc('Admin Message Received.Total') handled, result = self._handleReplicatorMessages(envelope) if handled: return result if isinstance(envelope.message, (ActorSystemMessage, logging.LogRecord)): thesplog('Admin of %s', envelope.identify(), level=logging.DEBUG) return getattr(self, 'h_' + envelope.message.__class__.__name__, self.unrecognized)(envelope) # else discard random non-admin messages self._sCBStats.inc('Admin Message Received.Ignored') thesplog('ADMIN DISCARD %s', envelope.identify(), level=logging.WARNING) return True def unrecognized(self, envelope): self._sCBStats.inc('Admin Message Received.Discarded') thesplog( "Admin got incoming %s from %s;" " discarded because I don't know how to handle it!", envelope.message, envelope.sender, level=logging.WARNING, primary=True) return True def isShuttingDown(self): return hasattr(self, '_exiting') def h_QueryExists(self, envelope): self._sCBStats.inc('Admin Message Received.Type.QueryExists') self._send_intent( TransmitIntent( envelope.sender, QueryAck( self.capabilities.get('Thespian ActorSystem Name', 'misc Actor System'), self.capabilities.get('Thespian ActorSystem Version', 'unknown Version'), self.isShuttingDown()))) return True def getStatus(self): resp = Thespian_SystemStatus(self.myAddress, capabilities=self.capabilities, inShutdown=self.isShuttingDown()) resp.setDeadLetterHandler(self._deadLetterHandler) self._updateStatusResponse(resp) resp.setLoadedSources(self._sources) resp.sourceAuthority = self._sourceAuthority return resp def h_Thespian_StatusReq(self, envelope): self._sCBStats.inc('Admin Message Received.Type.StatusReq') self._send_intent(TransmitIntent(envelope.sender, self.getStatus())) return True def thesplogStatus(self): "Write status to thesplog" from io import StringIO sd = StringIO() from thespian.system.messages.status import formatStatus try: sd.write('') SSD = lambda v: v except TypeError: class SSD(object): def __init__(self, sd): self.sd = sd def write(self, str_arg): self.sd.write(str_arg.decode('utf-8')) formatStatus(self.getStatus(), tofd=SSD(sd)) thesplog('STATUS: %s', sd.getvalue()) def h_SetLogging(self, envelope): return self.setLoggingControls(envelope) def h_SystemShutdown(self, envelope): self._exiting = envelope.sender thesplog('---- shutdown initiated by %s', envelope.sender, level=logging.DEBUG) # Send failure notices and clear out any pending children. If # any pending child ready notifications are received after # this, they will automatically be sent an ActorExitRequest. for each in self._pendingChildren: pendingReq = self._pendingChildren[each] self._send_intent( TransmitIntent( pendingReq.sender, PendingActorResponse(pendingReq.message.forActor, pendingReq.message.instanceNum, pendingReq.message.globalName, errorCode=PendingActorResponse. ERROR_ActorSystem_Shutting_Down))) self._pendingChildren = [] if not self.childAddresses: # no children? self._sayGoodbye() self.transport.abort_run(drain=True) return True # Now shutdown any direct children self._killLocalActors() # Callback will shutdown the Admin Once the children confirm # their exits. return True def _remove_expired_sources(self): rmvlist = [] ct = currentTime() for each in self._sources: if not self._sources[each].source_valid and \ self._sources[each].load_expires.view(ct).expired(): rmvlist.append(each) for each in rmvlist: self._cancel_pending_actors(self._sources[each].pending_actors) del self._sources[each] def _cancel_pending_actors( self, pending_envelopes, error_code=PendingActorResponse.ERROR_Invalid_SourceHash): for each in pending_envelopes: self._sendPendingActorResponse(each, None, errorCode=error_code) def _killLocalActors(self): for each in self.childAddresses: self._send_intent( TransmitIntent( each, ActorExitRequest(recursive=True), onError=lambda r, m, a=each: self._handleChildExited(a))) def _sayGoodbye(self): self._cleanupAdmin(self._reallySayGoodbye) def _reallySayGoodbye(self): self._send_intent( TransmitIntent( self._exiting, SystemShutdownCompleted(), onSuccess=lambda _i, _s, t=self.transport: t.abort_run(), onError=lambda _i, _s, t=self.transport: t.abort_run())) thesplog('---- shutdown completed', level=logging.INFO) logging.info('---- Actor System shutdown') self.shutdown_completed = True def h_ChildActorExited(self, envelope): return self._handleChildExited(envelope.message.childAddress) def _handleChildExited(self, childAddress): self._sourceNotifications = list( filter(lambda a: a != childAddress, self._sourceNotifications)) parentAddr = self._nannying.find(childAddress) if parentAddr: self._nannying.rmv(childAddress) # Let original requesting Actor (that *thinks* it's the # parent) know about this child exit as well. self._send_intent( TransmitIntent(parentAddr, ChildActorExited(childAddress))) return super(AdminCore, self)._handleChildExited(childAddress) def h_PendingActor(self, envelope): """Admin is creating an Actor. This covers one of the following cases: 1. Creating for external requester via ActorSystem. envelope.message.forActor will be None The Admin is the "parent" for all externally-created Actors. 2. Creating for another Actor when direct creation fails. Usually means that the current ActorSyste cannot meet the new Actor's requirements and the Admin should find another convention member that can create the new Actor. 3. GlobalName Actor. The Admin is the "parent" for all GlobalName Actors. """ self._sCBStats.inc('Admin Message Received.Type.Pending Actor Request') sourceHash = envelope.message.sourceHash thesplog('Pending Actor request received for %s%s reqs %s from %s', envelope.message.actorClassName, ' (%s)' % sourceHash if sourceHash else '', envelope.message.targetActorReq, envelope.sender) if self.isShuttingDown(): self._sendPendingActorResponse( envelope, None, errorCode=PendingActorResponse.ERROR_ActorSystem_Shutting_Down) return True sourceLoad = self._get_source_for_hash(sourceHash, envelope) if sourceLoad is True or sourceLoad is False: return sourceLoad # Note, both Admin and remote requester will have a local # child address for the child (with a different instanceNumber # each). childAddr = self._addrManager.createLocalAddress() childInstance = childAddr.addressDetails.addressInstanceNum self._pendingChildren[childInstance] = envelope try: self._startChildActor( childAddr, envelope.message.actorClassName, envelope.message.globalName, parentAddr=self.myAddress, # Admin is surrogate parent notifyAddr=self.myAddress, childRequirements=envelope.message.targetActorReq, sourceHash=sourceHash, sourceToLoad=sourceLoad) # transport will contrive to call _pendingActorReady when the # child is initialized and connected to this parent. return True except NoCompatibleSystemForActor as ex: thesplog(str(ex), level=logging.WARNING, primary=True) try: ret = self._not_compatible(envelope) if ret: return ret except NoCompatibleSystemForActor: pass errcode = PendingActorResponse.ERROR_No_Compatible_ActorSystem errstr = "" except InvalidActorSourceHash: errcode = PendingActorResponse.ERROR_Invalid_SourceHash errstr = "" except InvalidActorSpecification as ex: thesplog('Error: InvalidActorSpecification: %s', str(ex), exc_info=True) errcode = PendingActorResponse.ERROR_Invalid_ActorClass errstr = str(ex) except ImportError as ex: errcode = PendingActorResponse.ERROR_Import errstr = str(ex) except AttributeError as ex: # Usually when the module has no attribute FooActor thesplog('Error: AttributeError: %s', str(ex), exc_info=True) errcode = PendingActorResponse.ERROR_Invalid_ActorClass errstr = str(ex) except Exception as ex: import traceback thesplog('Exception "%s" handling PendingActor: %s', ex, traceback.format_exc(), level=logging.ERROR) errcode = PendingActorResponse.ERROR_Invalid_ActorClass errstr = str(ex) self._sendPendingActorResponse(envelope, None, errorCode=errcode, errorStr=errstr) self._retryPendingChildOperations(childInstance, None) return True def _not_compatible(self, createActorEnvelope): # Cannot do anything return False def _get_source_for_hash(self, sourceHash, createActorEnvelope): if not sourceHash: # Continue, with no source override return None self._remove_expired_sources() # If this source isn't known, see if there's some activity # that can be initiated to obtain the source. The expectation # is that the activity (if any) is asynchronous, so there's no # reason to continue processing here. if sourceHash not in self._sources: return self._get_missing_source_for_hash(sourceHash, createActorEnvelope) # If the source load is in-progress, save this request and # return with "continue" indication. if not self._sources[sourceHash].source_valid: self._sources[sourceHash].pending_actors.append( createActorEnvelope) return True # Return the requested source return self._sources[sourceHash] def _get_missing_source_for_hash(self, sourceHash, createActorEnvelope): # For a standard Actor System, if the source isn't currently # loaded then there's nothing that can be done, so reject the # createActor request and "continue". self._sendPendingActorResponse( createActorEnvelope, None, errorCode=PendingActorResponse.ERROR_Invalid_SourceHash) return True def _sendPendingActorResponse(self, requestEnvelope, actualAddress, errorCode=None, errorStr=None): # actualAddress is None for failure if actualAddress is None and errorCode is None: raise ValueError('Must specify either actualAddress or errorCode') self._send_intent( TransmitIntent( requestEnvelope.message.forActor or requestEnvelope.sender, PendingActorResponse(requestEnvelope.message.forActor, requestEnvelope.message.instanceNum, requestEnvelope.message.globalName, errorCode=errorCode, errorStr=errorStr, actualAddress=actualAddress))) def _pendingActorReady(self, childInstance, actualAddress): if childInstance not in self._pendingChildren: thesplog( 'Pending actor is ready at %s for UNKNOWN %s' '; sending child a shutdown', actualAddress, childInstance, level=logging.WARNING) self._send_intent( TransmitIntent(actualAddress, ActorExitRequest(recursive=True))) return requestEnvelope = self._pendingChildren[childInstance] del self._pendingChildren[childInstance] if requestEnvelope.message.globalName or \ not requestEnvelope.message.forActor: # The Admin is the responsible Parent for these children self._registerChild(actualAddress) else: # Anything the Admin was requested to create is a adoptive # child and should be killed when the Admin exits. self._registerChild(actualAddress) if requestEnvelope.message.forActor: # Proxy-parenting; remember the real parent self._nannying.add(actualAddress, requestEnvelope.message.forActor) self._addrManager.associateUseableAddress(self.myAddress, childInstance, actualAddress) # n.b. childInstance is for this Admin, but caller's # childInstance is in original request self._sendPendingActorResponse(requestEnvelope, actualAddress) self._retryPendingChildOperations(childInstance, actualAddress) def h_HandleDeadLetters(self, envelope): # handlerAddr, enableHandler self._sCBStats.inc( 'Admin Message Received.Type.Set Dead Letter Handler') if envelope.message.enableHandler: self._deadLetterHandler = envelope.message.handlerAddr else: if self._deadLetterHandler == envelope.message.handlerAddr: self._deadLetterHandler = None return True def h_DeadEnvelope(self, envelope): self._sCBStats.inc('Admin Message Received.Type.Dead Letter') if self._deadLetterHandler: self._send_intent( TransmitIntent(self._deadLetterHandler, envelope.message)) return True def h_RegisterSourceAuthority(self, envelope): self._sourceAuthority = envelope.message.authorityAddress def h_NotifyOnSourceAvailability(self, envelope): address = envelope.message.notificationAddress enable = envelope.message.enable all_except = [A for A in self._sourceNotifications if A != address] if enable: self._sourceNotifications = all_except + [address] for each in self._sources: if self._sources[each].source_valid: self._send_intent( TransmitIntent( address, LoadedSource(self._sources[each].srcHash, self._sources[each].srcInfo))) else: self._sourceNotifications = all_except def h_ValidateSource(self, envelope): self._remove_expired_sources() sourceHash = envelope.message.sourceHash if not envelope.message.sourceData: self.unloadActorSource(sourceHash) logging.getLogger('Thespian')\ .info('Source hash %s unloaded', sourceHash) return if sourceHash in self._sources and \ self._sources[sourceHash].source_valid: logging.getLogger('Thespian')\ .info('Source hash %s (%s) already loaded', sourceHash, self._sources[sourceHash].srcInfo if isinstance(self._sources[sourceHash], ValidSource) else '<pending>') return if self._sourceAuthority: self._sources[sourceHash] = PendingSource( sourceHash, envelope.message.sourceData) self._send_intent( TransmitIntent(self._sourceAuthority, envelope.message)) return # Any attempt to load sources is ignored if there is no active # Source Authority. This is a security measure to protect the # un-protected. logging.getLogger('Thespian').warning( 'No source authority to validate source hash %s', sourceHash) def h_ValidatedSource(self, envelope): self._remove_expired_sources() if envelope.sender != self._sourceAuthority: logging.getLogger('Thespian').warning( 'Ignoring validated source from %s: not the source authority at %s', envelope.sender, self._sourceAuthority) return source_hash = envelope.message.sourceHash if envelope.message.sourceZip: self._loadValidatedActorSource( source_hash, envelope.message.sourceZip, getattr(envelope.message, 'sourceInfo', None)) logging.getLogger('Thespian').info( 'Source hash %s (%s) validated by Source Authority' '; now available.', source_hash, getattr(envelope.message, 'sourceInfo', '-')) else: # Source Authority actively rejected this source, so # actively unloaded it. Alternatively the Source # Authority can do nothing and this load attempt will # timeout. self._cancel_pending_actors( self._sources[source_hash].pending_actors) logging.getLogger('Thespian').warning( 'Source hash %s (%s) REJECTED by Source Authority', source_hash, getattr(envelope.message, 'sourceInfo', '-')) del self._sources[source_hash] def _loadValidatedActorSource(self, sourceHash, sourceZip, sourceInfo): # Validate the source file; this doesn't actually utilize the # sourceZip, but it ensures that the sourceZip isn't garbage # before registering it as active source. if sourceHash not in self._sources: logging.getLogger('Thespian').warning( 'Provided validated source with no or expired request' ', hash %s; ignoring.', sourceHash) return try: f = SourceHashFinder(sourceHash, lambda v: v, sourceZip) namelist = f.getZipNames() logging.getLogger('Thespian').info( 'Validated source hash %s - %s, %s modules (%s)', sourceHash, sourceInfo, len(namelist), ', '.join(namelist if len(namelist) < 10 else namelist[:9] + ['...'])) except Exception as ex: logging.getLogger('Thespian')\ .error('Validated source (hash %s) is corrupted: %s', sourceHash, ex) return if self._sources[sourceHash].source_valid: # If a duplicate source load request is made while the # first is still being validated by the Source Authority, # another request will be sent to the Source Authority and # the latter response will be a duplicate here and can # simply be dropped. return # Store this registered source pending_actors = self._sources[sourceHash].pending_actors self._sources[sourceHash] = ValidSource( sourceHash, self._sources[sourceHash].orig_data, sourceZip, str(sourceInfo)) for each in pending_actors: self.h_PendingActor(each) msg = LoadedSource(self._sources[sourceHash].srcHash, self._sources[sourceHash].srcInfo) for each in self._sourceNotifications: self._send_intent(TransmitIntent(each, msg)) def unloadActorSource(self, sourceHash): if sourceHash in self._sources: msg = UnloadedSource(self._sources[sourceHash].srcHash, self._sources[sourceHash].srcInfo) for each in self._sourceNotifications: self._send_intent(TransmitIntent(each, msg)) del self._sources[sourceHash] for pnum, metapath in enumerate(sys.meta_path): if getattr(metapath, 'srcHash', None) == sourceHash: rmmods = [ M for M in sys.modules if M and M.startswith(metapath.hashRoot()) ] for each in rmmods: del sys.modules[each] del sys.meta_path[pnum] break def h_CapabilityUpdate(self, envelope): if self._updSystemCapabilities(envelope.message.capabilityName, envelope.message.capabilityValue): self._capUpdateLocalActors() def _updSystemCapabilities(self, cName, cVal): updateLocals = False if cVal is not None: updateLocals = cName not in self.capabilities or \ self.capabilities[cName] != cVal self.capabilities[cName] = cVal else: if cName in self.capabilities: updateLocals = True del self.capabilities[cName] return updateLocals def _capUpdateLocalActors(self): newCaps = NewCapabilities(self.capabilities, self.myAddress) for each in self.childAddresses: self._send_intent(TransmitIntent(each, newCaps))
class AdminCore(systemCommonBase): def __init__(self, transport, address, capabilities, logdefs, concurrency_context): thesplog('++++ Starting Admin from %s', sys.modules['thespian'].__file__, level=logging.DEBUG) super(AdminCore, self).__init__(address, transport) self.init_replicator(transport, concurrency_context) self.capabilities = capabilities self.logdefs = logdefs self._pendingChildren = {} # Use: childLocalAddr instance # : PendingActorEnvelope # Things that help us look like an Actor, even though we're not self._sourceHash = None thesplog('++++ Admin started @ %s / gen %s', self.transport.myAddress, str(ThespianGeneration), level=logging.INFO, primary=True) self._nannying = AssocList() # child actorAddress -> parent Address self._deadLetterHandler = None self._sources = {} # Index is sourcehash, value is requestor # ActorAddress or ValidSource (when validated) self._sourceAuthority = None def _activate(self): """Called when the full ActorSystem initialization is completed. This should then activate any functionality that needed to wait for completion of initialization. """ pass def run(self): try: while True: r = self.transport.run(self.handleIncoming, None) if isinstance(r, Thespian__UpdateWork): # tickle the transmit queues self._send_intent( TransmitIntent(self.myAddress, r)) continue # Expects that on completion of self.transport.run # that the Actor is done processing and that it has # been shutdown gracefully. break except Exception: import traceback thesplog('ActorAdmin uncaught exception: %s', traceback.format_exc(), level=logging.ERROR, exc_info=True) thesplog('Admin time to die', level=logging.DEBUG) def handleIncoming(self, envelope): self._sCBStats.inc('Admin Message Received.Total') handled, result = self._handleReplicatorMessages(envelope) if handled: return result if isinstance(envelope.message, (ActorSystemMessage, logging.LogRecord)): thesplog('Admin of %s', envelope.identify(), level=logging.DEBUG) return getattr(self, 'h_' + envelope.message.__class__.__name__, self.unrecognized)(envelope) # else discard random non-admin messages self._sCBStats.inc('Admin Message Received.Ignored') thesplog('ADMIN DISCARD %s', envelope.identify(), level=logging.WARNING) return True def unrecognized(self, envelope): self._sCBStats.inc('Admin Message Received.Discarded') thesplog("Admin got incoming %s from %s;" " discarded because I don't know how to handle it!", envelope.message, envelope.sender, level=logging.WARNING, primary=True) return True def isShuttingDown(self): return hasattr(self, '_exiting') def h_QueryExists(self, envelope): self._sCBStats.inc('Admin Message Received.Type.QueryExists') self._send_intent( TransmitIntent( envelope.sender, QueryAck( self.capabilities.get('Thespian ActorSystem Name', 'misc Actor System'), self.capabilities.get('Thespian ActorSystem Version', 'unknown Version'), self.isShuttingDown()))) return True def getStatus(self): resp = Thespian_SystemStatus(self.myAddress, capabilities=self.capabilities, inShutdown=self.isShuttingDown()) resp.setDeadLetterHandler(self._deadLetterHandler) self._updateStatusResponse(resp) resp.setLoadedSources(self._sources) resp.sourceAuthority = self._sourceAuthority return resp def h_Thespian_StatusReq(self, envelope): self._sCBStats.inc('Admin Message Received.Type.StatusReq') self._send_intent(TransmitIntent(envelope.sender, self.getStatus())) return True def thesplogStatus(self): "Write status to thesplog" from io import StringIO sd = StringIO() from thespian.system.messages.status import formatStatus try: sd.write('') SSD = lambda v: v except TypeError: class SSD(object): def __init__(self, sd): self.sd = sd def write(self, str_arg): self.sd.write(str_arg.decode('utf-8')) formatStatus(self.getStatus(), tofd=SSD(sd)) thesplog('STATUS: %s', sd.getvalue()) def h_SetLogging(self, envelope): return self.setLoggingControls(envelope) def h_SystemShutdown(self, envelope): self._exiting = envelope.sender thesplog('---- shutdown initiated by %s', envelope.sender, level=logging.DEBUG) # Send failure notices and clear out any pending children. If # any pending child ready notifications are received after # this, they will automatically be sent an ActorExitRequest. for each in self._pendingChildren: pendingReq = self._pendingChildren[each] self._send_intent( TransmitIntent( pendingReq.sender, PendingActorResponse( pendingReq.message.forActor, pendingReq.message.instanceNum, pendingReq.message.globalName, errorCode=PendingActorResponse.ERROR_ActorSystem_Shutting_Down))) self._pendingChildren = [] if not self.childAddresses: # no children? self._sayGoodbye() self.transport.abort_run(drain=True) return True # Now shutdown any direct children self._killLocalActors() # Callback will shutdown the Admin Once the children confirm # their exits. return True def _killLocalActors(self): for each in self.childAddresses: self._send_intent( TransmitIntent(each, ActorExitRequest(recursive=True), onError=lambda r, m, a=each: self._handleChildExited(a))) def _sayGoodbye(self): self._cleanupAdmin() self._send_intent(TransmitIntent(self._exiting, SystemShutdownCompleted())) thesplog('---- shutdown completed', level=logging.INFO) def h_ChildActorExited(self, envelope): return self._handleChildExited(envelope.message.childAddress) def _handleChildExited(self, childAddress): parentAddr = self._nannying.find(childAddress) if parentAddr: self._nannying.rmv(childAddress) # Let original requesting Actor (that *thinks* it's the # parent) know about this child exit as well. self._send_intent(TransmitIntent(parentAddr, ChildActorExited(childAddress))) return super(AdminCore, self)._handleChildExited(childAddress) def h_PendingActor(self, envelope): """Admin is creating an Actor. This covers one of the following cases: 1. Creating for external requester via ActorSystem. envelope.message.forActor will be None The Admin is the "parent" for all externally-created Actors. 2. Creating for another Actor when direct creation fails. Usually means that the current ActorSyste cannot meet the new Actor's requirements and the Admin should find another convention member that can create the new Actor. 3. GlobalName Actor. The Admin is the "parent" for all GlobalName Actors. """ self._sCBStats.inc('Admin Message Received.Type.Pending Actor Request') if self.isShuttingDown(): self._sendPendingActorResponse( envelope, None, errorCode=PendingActorResponse.ERROR_ActorSystem_Shutting_Down) return True sourceHash = envelope.message.sourceHash if sourceHash: if sourceHash not in self._sources: self._sendPendingActorResponse(envelope, None, errorCode = PendingActorResponse.ERROR_Invalid_SourceHash) return True if not isinstance(self._sources[sourceHash], ValidSource): thesplog('sourceHash %s is not valid (yet)', sourceHash) self._sendPendingActorResponse(envelope, None, errorCode = PendingActorResponse.ERROR_Invalid_SourceHash) return True # Note, both Admin and remote requester will have a local # child address for the child (with a different instanceNumber # each). childAddr = self._addrManager.createLocalAddress() childInstance = childAddr.addressDetails.addressInstanceNum try: self._startChildActor( childAddr, envelope.message.actorClassName, parentAddr=self.myAddress, # Admin is surrogate parent notifyAddr=self.myAddress, childRequirements=envelope.message.targetActorReq, sourceHash=sourceHash, sourceToLoad=(self._sources[sourceHash] if sourceHash else None)) except NoCompatibleSystemForActor: self._sendPendingActorResponse( envelope, None, errorCode=PendingActorResponse.ERROR_No_Compatible_ActorSystem) self._retryPendingChildOperations(childInstance, None) return True self._pendingChildren[childInstance] = envelope # transport will contrive to call _pendingActorReady when the # child is initialized and connected to this parent. return True def _sendPendingActorResponse(self, requestEnvelope, actualAddress, errorCode=None, errorStr=None): # actualAddress is None for failure if actualAddress is None and errorCode is None: raise ValueError('Must specify either actualAddress or errorCode') self._send_intent( TransmitIntent( requestEnvelope.message.forActor or requestEnvelope.sender, PendingActorResponse(requestEnvelope.message.forActor, requestEnvelope.message.instanceNum, requestEnvelope.message.globalName, errorCode=errorCode, errorStr=errorStr, actualAddress=actualAddress))) def _pendingActorReady(self, childInstance, actualAddress): if childInstance not in self._pendingChildren: thesplog('Pending actor is ready at %s for UNKNOWN %s' '; sending child a shutdown', actualAddress, childInstance, level=logging.WARNING) self._send_intent( TransmitIntent(actualAddress, ActorExitRequest(recursive=True))) return requestEnvelope = self._pendingChildren[childInstance] del self._pendingChildren[childInstance] if requestEnvelope.message.globalName or \ not requestEnvelope.message.forActor: # The Admin is the responsible Parent for these children self._registerChild(actualAddress) else: # Anything the Admin was requested to create is a adoptive # child and should be killed when the Admin exits. self._registerChild(actualAddress) if requestEnvelope.message.forActor: # Proxy-parenting; remember the real parent self._nannying.add(actualAddress, requestEnvelope.message.forActor) self._addrManager.associateUseableAddress(self.myAddress, childInstance, actualAddress) # n.b. childInstance is for this Admin, but caller's # childInstance is in original request self._sendPendingActorResponse(requestEnvelope, actualAddress) self._retryPendingChildOperations(childInstance, actualAddress) def h_HandleDeadLetters(self, envelope): # handlerAddr, enableHandler self._sCBStats.inc('Admin Message Received.Type.Set Dead Letter Handler') if envelope.message.enableHandler: self._deadLetterHandler = envelope.message.handlerAddr else: if self._deadLetterHandler == envelope.message.handlerAddr: self._deadLetterHandler = None return True def h_DeadEnvelope(self, envelope): self._sCBStats.inc('Admin Message Received.Type.Dead Letter') if self._deadLetterHandler: self._send_intent( TransmitIntent(self._deadLetterHandler, envelope.message)) return True def h_RegisterSourceAuthority(self, envelope): self._sourceAuthority = envelope.message.authorityAddress def h_ValidateSource(self, envelope): sourceHash = envelope.message.sourceHash if not envelope.message.sourceData: self.unloadActorSource(sourceHash) logging.getLogger('Thespian')\ .info('Source hash %s unloaded', sourceHash) return if sourceHash in self._sources: logging.getLogger('Thespian')\ .info('Source hash %s (%s) already loaded', sourceHash, self._sources[sourceHash].srcInfo if isinstance(self._sources[sourceHash], ValidSource) else '<pending>') return if self._sourceAuthority: self._send_intent(TransmitIntent(self._sourceAuthority, envelope.message)) return # Any attempt to load sources is ignored if there is no active # Source Authority. This is a security measure to protect the # un-protected. logging.getLogger('Thespian').warning( 'No source authority to validate source hash %s', sourceHash) def h_ValidatedSource(self, envelope): self._loadValidatedActorSource(envelope.message.sourceHash, envelope.message.sourceZip, getattr(envelope.message, 'sourceInfo', None)) thesplog('Source hash %s (%s) validated by Source Authority; now available.', envelope.message.sourceHash, getattr(envelope.message, 'sourceInfo', '-')) def _loadValidatedActorSource(self, sourceHash, sourceZip, sourceInfo): # Validate the source file; this doesn't actually utilize the # sourceZip, but it ensures that the sourceZip isn't garbage # before registering it as active source. try: f = SourceHashFinder(sourceHash, lambda v: v, sourceZip) namelist = f.getZipNames() logging.getLogger('Thespian').info( 'Validated source hash %s - %s, %s modules (%s)', sourceHash, sourceInfo, len(namelist), ', '.join(namelist if len(namelist) < 10 else namelist[:9] + ['...'])) except Exception as ex: logging.getLogger('Thespian')\ .error('Validated source (hash %s) is corrupted: %s', sourceHash, ex) return # Store this registered source self._sources[sourceHash] = ValidSource(sourceHash, sourceZip, str(sourceInfo)) def unloadActorSource(self, sourceHash): if sourceHash in self._sources: del self._sources[sourceHash] for pnum, metapath in enumerate(sys.meta_path): if getattr(metapath, 'srcHash', None) == sourceHash: rmmods = [M for M in sys.modules if M and M.startswith(metapath.hashRoot())] for each in rmmods: del sys.modules[each] del sys.meta_path[pnum] break def h_CapabilityUpdate(self, envelope): if self._updSystemCapabilities(envelope.message.capabilityName, envelope.message.capabilityValue): self._capUpdateLocalActors() def _updSystemCapabilities(self, cName, cVal): updateLocals = False if cVal is not None: updateLocals = cName not in self.capabilities or \ self.capabilities[cName] != cVal self.capabilities[cName] = cVal else: if cName in self.capabilities: updateLocals = True del self.capabilities[cName] return updateLocals def _capUpdateLocalActors(self): newCaps = NewCapabilities(self.capabilities, self.myAddress) for each in self.childAddresses: self._send_intent(TransmitIntent(each, newCaps))
class ConventioneerAdmin(GlobalNamesAdmin): """Extends the AdminCore+GlobalNamesAdmin with ActorSystem Convention functionality to support multi-host configurations. """ def __init__(self, *args, **kw): super(ConventioneerAdmin, self).__init__(*args, **kw) self._conventionMembers = AssocList( ) # key=Remote Admin Addr, value=ConventionMemberData self._conventionRegistration = ExpiryTime(timedelta(seconds=0)) self._conventionNotificationHandlers = [] self._conventionAddress = None # Not a member; still could be a leader self._pendingSources = { } # key = sourceHash, value is array of PendingActor requests self._hysteresisSender = HysteresisDelaySender(self._send_intent) def _updateStatusResponse(self, resp): resp.setConventionLeaderAddress(self._conventionAddress) resp.setConventionRegisterTime(self._conventionRegistration) for each in self._conventionMembers.values(): resp.addConventioneer(each.remoteAddress, each.registryValid) resp.setNotifyHandlers(self._conventionNotificationHandlers) super(ConventioneerAdmin, self)._updateStatusResponse(resp) def _activate(self): self.setupConvention() @property def _conventionLost(self): "True if this system was part of a convention but are no longer" return getattr(self, '_conventionLeaderIsGone', False) def setupConvention(self): if self.isShuttingDown(): return if not self._conventionAddress: gCA = getattr(self.transport, 'getConventionAddress', lambda c: None) self._conventionAddress = gCA(self.capabilities) if self._conventionAddress == self.myAddress: self._conventionAddress = None if self._conventionAddress: thesplog('Admin registering with Convention @ %s (%s)', self._conventionAddress, 'first time' if getattr(self, '_conventionLeaderIsGone', True) else 're-registering', level=logging.INFO, primary=True) self._hysteresisSender.sendWithHysteresis( TransmitIntent(self._conventionAddress, ConventionRegister( self.myAddress, self.capabilities, getattr(self, '_conventionLeaderIsGone', True)), onSuccess=self._setupConventionCBGood, onError=self._setupConventionCBError)) self._conventionRegistration = ExpiryTime( CONVENTION_REREGISTRATION_PERIOD) def _setupConventionCBGood(self, result, finishedIntent): self._sCBStats.inc('Admin Convention Registered') self._conventionLeaderIsGone = False if hasattr(self, '_conventionLeaderMissCount'): delattr(self, '_conventionLeaderMissCount') if getattr(self, 'asLogger', None): thesplog('Setting log aggregator of %s to %s', self.asLogger, self._conventionAddress) self.transport.scheduleTransmit( None, TransmitIntent(self.asLogger, LogAggregator(self._conventionAddress))) def _setupConventionCBError(self, result, finishedIntent): self._sCBStats.inc('Admin Convention Registration Failed') if hasattr(self, '_conventionLeaderMissCount'): self._conventionLeaderMissCount += 1 else: self._conventionLeaderMissCount = 1 if self._conventionLeaderMissCount < CONVENTION_REGISTRATION_MISS_MAX: thesplog( 'Admin cannot register with convention @ %s (miss %d): %s', finishedIntent.targetAddr, self._conventionLeaderMissCount, result, level=logging.WARNING, primary=True) else: thesplog('Admin convention registration lost @ %s (miss %d): %s', finishedIntent.targetAddr, self._conventionLeaderMissCount, result, level=logging.ERROR, primary=True) if hasattr(self, '_conventionLeaderIsGone'): self._conventionLeaderIsGone = True def isConventionLeader(self): return self._conventionAddress is None or self._conventionAddress == self.myAddress def h_ConventionInvite(self, envelope): self._conventionAddress = envelope.sender self.setupConvention() def h_ConventionRegister(self, envelope): if self.isShuttingDown(): return self._sCBStats.inc('Admin Handle Convention Registration') # Registrant may re-register if changing capabilities registrant = envelope.message.adminAddress thesplog( 'Got Convention registration from %s (%s) (new? %s)', registrant, 'first time' if envelope.message.firstTime else 're-registering', not self._conventionMembers.find(registrant), level=logging.INFO) if registrant == self.myAddress: # Either remote failed getting an external address and is # using 127.0.0.1 or else this is a malicious attempt to # make us talk to ourselves. Ignore it. thesplog( 'Convention registration from %s is an invalid address; ignoring.', registrant, level=logging.WARNING) return True if envelope.message.firstTime: # erase knowledge of actors associated with potential # former instance of this system self._remoteSystemCleanup(registrant) if registrant == self._conventionAddress: self._conventionLeaderIsGone = False existing = self._conventionMembers.find(registrant) if existing: existing.refresh(envelope.message.capabilities) else: newmember = ConventionMemberData(registrant, envelope.message.capabilities) if getattr(envelope.message, 'preRegister', False): # getattr used; see definition newmember.preRegistered = PreRegistration( ) # will attempt registration immediately self._conventionMembers.add(registrant, newmember) if self.isConventionLeader(): if not existing: for each in self._conventionNotificationHandlers: self._send_intent( TransmitIntent( each, ActorSystemConventionUpdate( registrant, envelope.message.capabilities, True))) # errors ignored # If we are the Convention Leader, this would be the point to # inform all other registrants of the new registrant. At # present, there is no reciprocity here, so just update the # new registrant with the leader's info. self._send_intent( TransmitIntent( registrant, ConventionRegister( self.myAddress, self.capabilities, # first time in, then must be first time out envelope.message.firstTime))) return True def h_ConventionDeRegister(self, envelope): self._sCBStats.inc('Admin Handle Convention De-registration') remoteAdmin = envelope.message.adminAddress if remoteAdmin == self.myAddress: # Either remote failed getting an external address and is # using 127.0.0.1 or else this is a malicious attempt to # make us talk to ourselves. Ignore it. thesplog( 'Convention deregistration from %s is an invalid address; ignoring.', remoteAdmin, level=logging.WARNING) return True if getattr(envelope.message, 'preRegistered', False): # see definition for getattr use existing = self._conventionMembers.find(remoteAdmin) if existing: existing.preRegistered = None self._remoteSystemCleanup(remoteAdmin) return True def h_SystemShutdown(self, envelope): if self._conventionAddress: thesplog('Admin de-registering with Convention @ %s', str(self._conventionAddress), level=logging.INFO, primary=True) self._hysteresisSender.cancelSends(self._conventionAddress) self._send_intent( TransmitIntent(self._conventionAddress, ConventionDeRegister(self.myAddress))) else: for each in self._conventionMembers.values(): rmtaddr = each.remoteAddress self._hysteresisSender.cancelSends(rmtaddr) self._send_intent( TransmitIntent(rmtaddr, ConventionDeRegister(self.myAddress))) return super(ConventioneerAdmin, self).h_SystemShutdown(envelope) def _remoteSystemCleanup(self, registrant): """Called when a RemoteActorSystem has exited and all associated Actors should be marked as exited and the ActorSystem removed from Convention membership. """ thesplog('Convention cleanup or deregistration for %s (new? %s)', registrant, not self._conventionMembers.find(registrant), level=logging.INFO) if hasattr(self.transport, 'lostRemote'): self.transport.lostRemote(registrant) cmr = self._conventionMembers.find(registrant) if cmr: # Send exited notification to conventionNotificationHandler (if any) if self.isConventionLeader(): for each in self._conventionNotificationHandlers: self._send_intent( TransmitIntent( each, ActorSystemConventionUpdate( cmr.remoteAddress, cmr.remoteCapabilities, False))) # errors ignored # If the remote ActorSystem shutdown gracefully (i.e. sent # a Convention Deregistration) then it should not be # necessary to shutdown remote Actors (or notify of their # shutdown) because the remote ActorSystem should already # have caused this to occur. However, it won't hurt, and # it's necessary if the remote ActorSystem did not exit # gracefully. for lpa, raa in cmr.hasRemoteActors: # ignore errors: self._send_intent(TransmitIntent(lpa, ChildActorExited(raa))) # n.b. at present, this means that the parent might # get duplicate notifications of ChildActorExited; it # is expected that Actors can handle this. # Remove remote system from conventionMembers if not cmr.preRegistered: self._conventionMembers.rmv(registrant) else: # This conventionMember needs to stay because the # current system needs to continue issuing # registration pings. By setting the registryValid # expiration to forever, this member won't re-time-out # and will therefore be otherwise ignored... until it # registers again at which point the membership will # be updated with new settings. cmr.registryValid = ExpiryTime(None) if registrant == self._conventionAddress: # Convention Leader has exited. Do NOT set # conventionAddress to None. It might speed up shutdown # of this ActorSystem because it won't try to de-register # from the convention leader, but if the convention leader # reappears there will be nothing to get this ActorSystem # re-registered with the convention. self._conventionLeaderIsGone = True self._hysteresisSender.cancelSends(registrant) def _checkConvention(self): if self.isConventionLeader(): missing = [] for each in self._conventionMembers.values(): if each.registryValid.expired(): missing.append(each) for each in missing: thesplog('%s missed %d checkins (%s); assuming it has died', str(each), CONVENTION_REGISTRATION_MISS_MAX, str(each.registryValid), level=logging.WARNING, primary=True) self._remoteSystemCleanup(each.remoteAddress) self._conventionRegistration = ExpiryTime( CONVENTION_REREGISTRATION_PERIOD) else: # Re-register with the Convention if it's time if self._conventionAddress and self._conventionRegistration.expired( ): self.setupConvention() for member in self._conventionMembers.values(): if member.preRegistered and \ member.preRegistered.pingValid.expired() and \ not member.preRegistered.pingPending: member.preRegistered.pingPending = True member.preRegistered.pingValid = ExpiryTime( CONVENTION_RESTART_PERIOD if member.registryValid.expired( ) else CONVENTION_REREGISTRATION_PERIOD) self._hysteresisSender.sendWithHysteresis( TransmitIntent(member.remoteAddress, ConventionInvite(), onSuccess=self._preRegQueryNotPending, onError=self._preRegQueryNotPending)) def _preRegQueryNotPending(self, result, finishedIntent): remoteAddr = finishedIntent.targetAddr member = self._conventionMembers.find(remoteAddr) if member and member.preRegistered: member.preRegistered.pingPending = False def run(self): # Main loop for convention management. Wraps the lower-level # transport with a stop at the next needed convention # registration period to re-register. try: while not self.isShuttingDown(): delay = min(self._conventionRegistration or \ ExpiryTime(CONVENTION_RESTART_PERIOD if self._conventionLost and not self.isConventionLeader() else CONVENTION_REREGISTRATION_PERIOD), ExpiryTime(None) if self._hysteresisSender.delay.expired() else self._hysteresisSender.delay ) # n.b. delay does not account for soon-to-expire # pingValids, but since delay will not be longer than # a CONVENTION_REREGISTRATION_PERIOD, the worst case # is a doubling of a pingValid period (which should be fine). self.transport.run(self.handleIncoming, delay.remaining()) self._checkConvention() self._hysteresisSender.checkSends() except Exception as ex: import traceback thesplog('ActorAdmin uncaught exception: %s', traceback.format_exc(), level=logging.ERROR, exc_info=True) thesplog('Admin time to die', level=logging.DEBUG) # ---- Source Hash Transfers -------------------------------------------------- def h_SourceHashTransferRequest(self, envelope): sourceHash = envelope.message.sourceHash src = self._sources.get(sourceHash, None) self._send_intent( TransmitIntent( envelope.sender, SourceHashTransferReply(sourceHash, src and src.zipsrc, src and src.srcInfo))) return True def h_SourceHashTransferReply(self, envelope): sourceHash = envelope.message.sourceHash pending = self._pendingSources[sourceHash] del self._pendingSources[sourceHash] if envelope.message.isValid(): self._sources[sourceHash] = ValidSource( sourceHash, envelope.message.sourceData, getattr(envelope.message, 'sourceInfo', None)) for each in pending: self.h_PendingActor(each) else: for each in pending: self._sendPendingActorResponse( each, None, errorCode=PendingActorResponse.ERROR_Invalid_SourceHash) return True def h_ValidateSource(self, envelope): if not envelope.message.sourceData and envelope.sender != self._conventionAddress: # Propagate source unload requests to all convention members for each in self._conventionMembers.values(): # Do not propagate if this is where the notification # came from; prevents indefinite bouncing of this # message as long as the convention structure is a # DAG. if each.remoteAddress != envelope.sender: self._hysteresisSender.sendWithHysteresis( TransmitIntent(each.remoteAddress, envelope.message)) super(ConventioneerAdmin, self).h_ValidateSource(envelope) return False # might have sent with hysteresis, so break out to local _run def _sentByRemoteAdmin(self, envelope): for each in self._conventionMembers.values(): if envelope.sender == each.remoteAddress: return True return False def _acceptsRemoteLoadedSourcesFrom(self, pendingActorEnvelope): allowed = self.capabilities.get('AllowRemoteActorSources', 'yes') return allowed.lower() == 'yes' or \ (allowed == 'LeaderOnly' and pendingActorEnvelope.sender == self._conventionAddress) # ---- Remote Actor interactions ---------------------------------------------- def h_PendingActor(self, envelope): sourceHash = envelope.message.sourceHash childRequirements = envelope.message.targetActorReq thesplog('Pending Actor request received for %s%s reqs %s from %s', envelope.message.actorClassName, ' (%s)' % sourceHash if sourceHash else '', childRequirements, envelope.sender) # If this request was forwarded by a remote Admin and the # sourceHash is not known locally, request it from the sending # remote Admin if sourceHash and \ sourceHash not in self._sources and \ self._sentByRemoteAdmin(envelope) and \ self._acceptsRemoteLoadedSourcesFrom(envelope): requestedAlready = self._pendingSources.get(sourceHash, False) self._pendingSources.setdefault(sourceHash, []).append(envelope) if not requestedAlready: self._hysteresisSender.sendWithHysteresis( TransmitIntent(envelope.sender, SourceHashTransferRequest(sourceHash))) return False # sent with hysteresis, so break out to local _run return True # If the requested ActorClass is compatible with this # ActorSystem, attempt to start it, otherwise forward the # request to any known compatible ActorSystem. try: childClass = actualActorClass( envelope.message.actorClassName, partial(loadModuleFromHashSource, sourceHash, self._sources) if sourceHash else None) acceptsCaps = lambda caps: checkActorCapabilities( childClass, caps, childRequirements) if not acceptsCaps(self.capabilities): if envelope.message.forActor is None: # Request from external; use sender address envelope.message.forActor = envelope.sender remoteCandidates = [ K for K in self._conventionMembers.values() if not K.registryValid.expired() and K.remoteAddress != envelope.sender # source Admin and K.remoteAddress not in getattr(envelope.message, 'alreadyTried', []) and acceptsCaps(K.remoteCapabilities) ] if not remoteCandidates: if self.isConventionLeader(): thesplog( 'No known ActorSystems can handle a %s for %s', childClass, envelope.message.forActor, level=logging.WARNING, primary=True) self._sendPendingActorResponse( envelope, None, errorCode=PendingActorResponse. ERROR_No_Compatible_ActorSystem) return True # Let the Convention Leader try to find an appropriate ActorSystem bestC = self._conventionAddress else: # distribute equally amongst candidates C = [(K.remoteAddress, len(K.hasRemoteActors)) for K in remoteCandidates] bestC = foldl( lambda best, possible: best if best[1] <= possible[1] else possible, C)[0] thesplog('Requesting creation of %s%s on remote admin %s', envelope.message.actorClassName, ' (%s)' % sourceHash if sourceHash else '', bestC) envelope.message.alreadyTried.append(self.myAddress) self._send_intent(TransmitIntent(bestC, envelope.message)) return True except InvalidActorSourceHash: self._sendPendingActorResponse( envelope, None, errorCode=PendingActorResponse.ERROR_Invalid_SourceHash) return True except InvalidActorSpecification: self._sendPendingActorResponse( envelope, None, errorCode=PendingActorResponse.ERROR_Invalid_ActorClass) return True except ImportError as ex: self._sendPendingActorResponse( envelope, None, errorCode=PendingActorResponse.ERROR_Import, errorStr=str(ex)) return True except AttributeError as ex: # Usually when the module has no attribute FooActor self._sendPendingActorResponse( envelope, None, errorCode=PendingActorResponse.ERROR_Invalid_ActorClass, errorStr=str(ex)) return True except Exception as ex: import traceback thesplog('Exception "%s" handling PendingActor: %s', ex, traceback.format_exc(), level=logging.ERROR) self._sendPendingActorResponse( envelope, None, errorCode=PendingActorResponse.ERROR_Invalid_ActorClass, errorStr=str(ex)) return True return super(ConventioneerAdmin, self).h_PendingActor(envelope) def h_NotifyOnSystemRegistration(self, envelope): if envelope.message.enableNotification: newRegistrant = envelope.sender not in self._conventionNotificationHandlers if newRegistrant: self._conventionNotificationHandlers.append(envelope.sender) # Now update the registrant on the current state of all convention members for member in self._conventionMembers.values(): self._send_intent( TransmitIntent( envelope.sender, ActorSystemConventionUpdate( member.remoteAddress, member.remoteCapabilities, True))) else: self._removeNotificationHandler(envelope.sender) return True def _removeNotificationHandler(self, handlerAddr): self._conventionNotificationHandlers = [ H for H in self._conventionNotificationHandlers if H != handlerAddr ] def h_PoisonMessage(self, envelope): self._removeNotificationHandler(envelope.sender) def _handleChildExited(self, childAddr): self._removeNotificationHandler(childAddr) return super(ConventioneerAdmin, self)._handleChildExited(childAddr) def h_CapabilityUpdate(self, envelope): updateLocals = self._updSystemCapabilities( envelope.message.capabilityName, envelope.message.capabilityValue) self.setupConvention() if updateLocals: self._capUpdateLocalActors() return False # might have sent with Hysteresis, so return to _run loop here
class MultiprocessQueueTransport(asyncTransportBase, wakeupTransportBase): """A transport designed to use a multiprocess.Queue instance to send and receive messages with other multiprocess Process actors. There is one instance of this object in each Actor. This object maintains a single input queue (used by its parent an any children it creates) and a table of all known sub-Actor addresses and their queues (being the most immediate child Actor queue that moves the message closer to the child target, or the Parent actor queue to which the message should be forwarded if no child is identified). """ def __init__(self, initType, *args): super(MultiprocessQueueTransport, self).__init__() if isinstance(initType, ExternalInterfaceTransportInit): # External process that's going to talk "in". There is no # parent, and the child is the systemAdmin. capabilities, logDefs, self._concontext = args self._parentQ = None NewQ = self._concontext.Queue if self._concontext else Queue self._adminQ = NewQ(MAX_ADMIN_QUEUESIZE) self._adminAddr = self.getAdminAddr(capabilities) self._myQAddress = ActorAddress(QueueActorAddress('~')) self._myInputQ = NewQ(MAX_ACTOR_QUEUESIZE) elif isinstance(initType, MpQTEndpoint): _addrInst, myAddr, myQueue, parentQ, adminQ, adminAddr, ccon = initType.args self._concontext = ccon self._parentQ = parentQ self._adminQ = adminQ self._adminAddr = adminAddr self._myQAddress = myAddr self._myInputQ = myQueue else: thesplog('MultiprocessQueueTransport init of type %s unsupported!', str(initType), level=logging.ERROR) # _queues is a map of direct child ActorAddresses to Queue instance. Note # that there will be multiple keys mapping to the same Queue # instance because routing is only either to the Parent or to # an immediate Child. self._queues = AssocList() # addr -> queue # _fwdvia represents routing for other than immediate parent # or child (there may be multiple target addresses mapping to # the same forward address. self._fwdvia = AssocList() # targetAddress -> fwdViaAddress self._nextSubInstance = 0 def protectedFileNumList(self): return foldl(lambda a, b: a + [b._reader.fileno( ), b._writer.fileno()], [self._myInputQ, self._parentQ, self._adminQ] + list(self._queues.values()), []) def childResetFileNumList(self): return foldl(lambda a, b: a + [b._reader.fileno(), b._writer.fileno()], [self._parentQ] + list(self._queues.values()), []) @property def myAddress(self): return self._myQAddress @staticmethod def getAddressFromString(addrspec): # addrspec is assumed to be a valid address string return ActorAddress(QueueActorAddress(addrspec)) @staticmethod def getAdminAddr(capabilities): return MultiprocessQueueTransport.getAddressFromString( capabilities.get('Admin Address', 'ThespianQ')) @staticmethod def probeAdmin(addr): """Called to see if there might be an admin running already at the specified addr. This is called from the systemBase, so simple blocking operations are fine. This only needs to check for a responder; higher level logic will verify that it's actually an ActorAdmin suitable for use. """ # never reconnectable; Queue objects are only available from # the constructor and cannot be synthesized or passed. return False def _updateStatusResponse(self, resp): "Called to update a Thespian_SystemStatus or Thespian_ActorStatus with common information" asyncTransportBase._updateStatusResponse(self, resp) wakeupTransportBase._updateStatusResponse(self, resp) def _nextSubAddress(self): subAddrStr = self._myQAddress.addressDetails.subAddr( self._nextSubInstance) self._nextSubInstance = self._nextSubInstance + 1 return ActorAddress(QueueActorAddress(subAddrStr)) def prepEndpoint(self, assignedLocalAddr, capabilities): """In the parent, prepare to establish a new communications endpoint with a new Child Actor. The result of this call will be passed to a created child process to use when initializing the Transport object for that class; the result of this call will also be kept by the parent to finalize the communications after creation of the Child by calling connectEndpoint() with this returned object. """ NewQ = self._concontext.Queue if self._concontext else Queue if isinstance(assignedLocalAddr.addressDetails, ActorLocalAddress): return MpQTEndpoint( assignedLocalAddr.addressDetails.addressInstanceNum, self._nextSubAddress(), NewQ(MAX_ACTOR_QUEUESIZE), self._myInputQ, self._adminQ, self._adminAddr, self._concontext) return MpQTEndpoint(None, assignedLocalAddr, self._adminQ, self._myInputQ, self._adminQ, self._adminAddr, self._concontext) def connectEndpoint(self, endPoint): """Called by the Parent after creating the Child to fully connect the endpoint to the Child for ongoing communications.""" (_addrInst, childAddr, childQueue, _myQ, _adminQ, _adminAddr, _concurrency_context) = endPoint.args self._queues.add(childAddr, childQueue) def deadAddress(self, addressManager, childAddr): # Can no longer send to this Queue object. Delete the # entry; this will cause forwarding of messages, although # the addressManager is also aware of the dead address and # will cause DeadEnvelope forwarding. Deleting here # prevents hanging on queue full to dead children. self._queues.rmv(childAddr) deadfwd, okfwd = ([],[]) if False else \ partition(lambda i: i[0] == childAddr or i[1] == childAddr, self._fwdvia.items()) if deadfwd: self._fwdvia = AssocList() for A, AQ in okfwd: self._fwdvia.add(A, AQ) super(MultiprocessQueueTransport, self).deadAddress(addressManager, childAddr) def _runWithExpiry(self, incomingHandler): """Core scheduling method; called by the current Actor process when idle to await new messages (or to do background processing). """ if incomingHandler == TransmitOnly or \ isinstance(incomingHandler, TransmitOnly): # transmits are not queued/multistage in this transport, no waiting return 0 self._aborting_run = False while not self.run_time.expired() and not self._aborting_run: try: rcvd = self._myInputQ.get(True, self.run_time.remainingSeconds()) except Q.Empty: # Probably a timeout, but let the while loop decide for sure continue if rcvd == 'BuMP': return Thespian__UpdateWork() relayAddr, (sendAddr, destAddr, msg) = rcvd if not self._queues.find(sendAddr): # We don't directly know about this sender, so # remember what path this arrived on to know where to # direct future messages for this sender. if relayAddr and self._queues.find(relayAddr) and \ not self._fwdvia.find(sendAddr): # relayAddr might be None if it's our parent, which is OK because # the default message forwarding is to the parent. If it's not # none, it should be in self._queues though! self._fwdvia.add(sendAddr, relayAddr) if hasattr(self, '_addressMgr'): destAddr, msg = self._addressMgr.prepMessageSend(destAddr, msg) if destAddr is None: thesplog('Unexpected target inaccessibility for %s', msg, level=logging.WARNING) raise CannotPickleAddress(destAddr) if msg is SendStatus.DeadTarget: thesplog( 'Faking message "sent" because target is dead and recursion avoided.' ) continue if destAddr == self._myQAddress: if incomingHandler is None: return ReceiveEnvelope(sendAddr, msg) if not incomingHandler(ReceiveEnvelope(sendAddr, msg)): return # handler returned False, indicating run() should exit else: # Note: the following code has implicit knowledge of serialize() and xmit putQValue = lambda relayer: (relayer, (sendAddr, destAddr, msg)) deadQValue = lambda relayer: (relayer, (sendAddr, self._adminAddr, DeadEnvelope(destAddr, msg))) # Must forward this packet via a known forwarder or our parent. tgtQ = self._queues.find(destAddr) if tgtQ: sendArgs = putQValue(self.myAddress), True if not tgtQ: tgtA = self._fwdvia.find(destAddr) if tgtA: tgtQ = self._queues.find(tgtA) sendArgs = putQValue(None), if tgtQ: try: tgtQ.put(*sendArgs, timeout=timePeriodSeconds( MAX_QUEUE_TRANSMIT_PERIOD)) except Q.Full: thesplog( 'Unable to send msg %s to dest %s; dead lettering', msg, destAddr) try: (self._parentQ or self._adminQ).put( deadQValue( self.myAddress if self._parentQ else None), True, timePeriodSeconds(MAX_QUEUE_TRANSMIT_PERIOD)) except Q.Full: thesplog( 'Unable to send deadmsg %s to %s or admin; discarding', msg, destAddr) else: # Not sure how to route this message yet. It # could be a heretofore silent child of one of our # children, it could be our parent (whose address # we don't know), or it could be elsewhere in the # tree. # # Try sending it to the parent first. If the # parent can't determine the routing, it will be # sent back down (relayAddr will be None in that # case) and it must be sprayed out to all children # in case the target lives somewhere beneath us. # Note that _parentQ will be None for top-level # actors, which send up to the Admin instead. # # As a special case, the external system is the # parent of the admin, but the admin is the # penultimate parent of all others, so this code # must keep the admin and the parent from playing # ping-pong with the message. But... the message # might be directed to the external system, which # is the parent of the Admin, so we need to check # with it first. # parentQ == None but adminQ good --> external # parentQ and adminQ and myAddress == adminAddr --> Admin # parentQ and adminQ and myAddress != adminADdr --> other Actor if relayAddr: # Send message up to the parent to see if the # parent knows how to forward it try: (self._parentQ or self._adminQ).put( putQValue( self.myAddress if self._parentQ else None), True, timePeriodSeconds(MAX_QUEUE_TRANSMIT_PERIOD)) except Q.Full: thesplog( 'Unable to send dead msg %s to %s or admin; discarding', msg, destAddr) else: # Sent by parent or we are an external, so this # may be some grandchild not currently known. # Do the worst case and just send this message # to ALL immediate children, hoping it will # get there via some path. for A, AQ in self._queues.items(): if A not in [ self._adminAddr, str(self._adminAddr) ]: # None means sent by Parent, so don't # send BACK to parent if unknown try: AQ.put( putQValue(None), True, timePeriodSeconds( MAX_QUEUE_TRANSMIT_PERIOD)) except Q.Full: pass return None def abort_run(self, drain=False): # Queue transmits immediately, so no draining needed self._aborting_run = True def serializer(self, intent): wrappedMsg = self._myQAddress, intent.targetAddr, intent.message # For multiprocess Queues, the serialization (pickling) of the # outbound message happens in a separate process. This is # unfortunate because if the message is not pickle-able, the # exception is thrown (and not handled) in the other process, # and this process has no indication of the issue. The # unfortunate solution is that pickling must be tried in the # current process first to detect these errors (unfortunate # because that means each message gets pickled twice, # impacting performance). discard = pickle.dumps(wrappedMsg) return wrappedMsg def interrupt_wait(self): self._myInputQ.put_nowait('BuMP') def _scheduleTransmitActual(self, transmitIntent): try: if transmitIntent.targetAddr == self.myAddress: if transmitIntent.message: self._myInputQ.put( (self._myQAddress, transmitIntent.serMsg), True, timePeriodSeconds(transmitIntent.delay())) else: tgtQ = self._queues.find(transmitIntent.targetAddr) if tgtQ: tgtQ.put((self._myQAddress, transmitIntent.serMsg), True, timePeriodSeconds(transmitIntent.delay())) else: # None means sent by parent, so don't send BACK to parent if unknown topOrFromBelow = self._myQAddress if self._parentQ else None (self._parentQ or self._adminQ).put( (topOrFromBelow, transmitIntent.serMsg), True, timePeriodSeconds(transmitIntent.delay())) transmitIntent.tx_done(SendStatus.Sent) return except Q.Full: pass transmitIntent.tx_done( SendStatus.DeadTarget if not isinstance(transmitIntent._message, ( ChildActorExited, ActorExitRequest)) else SendStatus.Failed) thesplog('Q.Full %s to %s result %s', transmitIntent._message, transmitIntent.targetAddr, transmitIntent.result)