예제 #1
0
 def __init__(self, transport, address, capabilities, logdefs,
              concurrency_context):
     thesplog('++++ Starting Admin from %s',
              sys.modules['thespian'].__file__,
              level=logging.DEBUG)
     super(AdminCore, self).__init__(address, transport)
     self.init_replicator(transport, concurrency_context)
     self.capabilities = capabilities
     self.logdefs = logdefs
     self._pendingChildren = {
     }  # Use: childLocalAddr instance # : PendingActorEnvelope
     # Things that help us look like an Actor, even though we're not
     self._sourceHash = None
     thesplog('++++ Admin started @ %s / gen %s',
              self.transport.myAddress,
              str(ThespianGeneration),
              level=logging.INFO,
              primary=True)
     logging.info('++++ Actor System gen %s started, admin @ %s',
                  str(ThespianGeneration), self.transport.myAddress)
     logging.debug('Thespian source: %s', sys.modules['thespian'].__file__)
     self._nannying = AssocList()  # child actorAddress -> parent Address
     self._deadLetterHandler = None
     self._sources = {
     }  # Index is sourcehash, value PendingSource or ValidSource
     self._sourceAuthority = None
     self._sourceNotifications = []  # array of notification addresses
예제 #2
0
 def unrecognized(self, envelope):
     self._sCBStats.inc('Admin Message Received.Discarded')
     thesplog("Admin got incoming %s from %s;"
              " discarded because I don't know how to handle it!",
              envelope.message, envelope.sender,
              level=logging.WARNING, primary=True)
     return True
예제 #3
0
    def __init__(self, initType, *args):
        super(UDPTransport, self).__init__()

        if isinstance(initType, ExternalInterfaceTransportInit):
            # External process that is going to talk "in".  There is
            # no parent, and the child is the systemAdmin.
            capabilities, logDefs = args
            templateAddr          = UDPv4ActorAddress(None, 0)
            self.socket           = socket.socket(*templateAddr.socketArgs)
            self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            self.socket.bind(*templateAddr.bindArgs)
            self.myAddress        = ActorAddress(UDPv4ActorAddress(*self.socket.getsockname(),
                                                                   external=True))
            thesplog('external template %s got actual %s', templateAddr, self.myAddress,
                     level=logging.DEBUG)
            self._adminAddr       = self.getAdminAddr(capabilities)
            self._parentAddr      = None
        elif isinstance(initType, UDPEndpoint):
            instanceNum, assignedAddr, self._parentAddr, self._adminAddr = initType.args
            templateAddr = assignedAddr or ActorAddress(UDPv4ActorAddress(None, 0))
            self.socket           = socket.socket(*templateAddr.addressDetails.socketArgs)
            self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            self.socket.bind(*templateAddr.addressDetails.bindArgs)
            # N.B.  myAddress is actually the address we will export
            # for others to talk to us, not the bind address.  The
            # difference is that we bind to '0.0.0.0' (inaddr_any),
            # but that's not a valid address for people to send stuff
            # to us.
            self.myAddress = ActorAddress(UDPv4ActorAddress(*self.socket.getsockname(),
                                                            external=True))
        else:
            thesplog('UDPTransport init of type %s unsupported', str(initType), level=logging.ERROR)
        self._rcvd = []
예제 #4
0
 def exec_module(self, module):
     moduleName = module.__name__
     hashRoot = self.finder.hashRoot()
     if moduleName.startswith(hashRoot):
         moduleName = moduleName[len(hashRoot):]
     if self.isModuleDir:
         name = ospath.join(*tuple(moduleName.split('.') + ['__init__.py']))
     elif '.' in moduleName:
         name = ospath.join(*tuple(moduleName.split('.'))) + '.py'
     else:
         name = moduleName + '.py'
     codeproc = lambda s: fix_imports(s, name, hashRoot, self.finder.getZipTopLevelNames())
     try:
         # Ensure the file ends in a carriage-return.  The path
         # importer does this automatically and no trailing
         # whitespace results in SyntaxError or IndentError
         # exceptions.  In addition, using "universal newlines"
         # mode to read the file is not always effective
         # (e.g. ntlm.HTTPNtlmAuthHandler.py, so explicitly ensure
         # the proper line endings for the compiler.
         if sys.version_info >= (3,0):
             converter = lambda s: codeproc(s + b'\n')
         else:
             converter = lambda s: codeproc(s.replace('\r\n', '\n')+'\n')
         code = self.finder.withZipElementSource(
             name,
             converter)
         do_exec(code, module.__dict__)
     except Exception as ex:
         thesplog('sourceload realization failure in %s: %s',
                  moduleName, ex, level=logging.ERROR)
         #return None
         raise
예제 #5
0
 def unrecognized(self, envelope):
     self._sCBStats.inc('Admin Message Received.Discarded')
     thesplog("Admin got incoming %s from %s;"
              " discarded because I don't know how to handle it!",
              envelope.message, envelope.sender,
              level=logging.WARNING, primary=True)
     return True
예제 #6
0
 def __init__(self, transport, address, capabilities,
              logdefs,
              concurrency_context):
     thesplog('++++ Starting Admin from %s',
              sys.modules['thespian'].__file__,
              level=logging.DEBUG)
     super(AdminCore, self).__init__(address, transport)
     self.init_replicator(transport, concurrency_context)
     self.capabilities = capabilities
     self.logdefs = logdefs
     self._pendingChildren = {}  # Use: childLocalAddr instance # : PendingActorEnvelope
     # Things that help us look like an Actor, even though we're not
     self._sourceHash = None
     thesplog('++++ Admin started @ %s / gen %s',
              self.transport.myAddress, str(ThespianGeneration),
              level=logging.INFO,
              primary=True)
     logging.info('++++ Actor System gen %s started, admin @ %s',
                  str(ThespianGeneration), self.transport.myAddress)
     logging.debug('Thespian source: %s', sys.modules['thespian'].__file__)
     self._nannying = AssocList()  # child actorAddress -> parent Address
     self._deadLetterHandler = None
     self._sources = {}  # Index is sourcehash, value PendingSource or ValidSource
     self._sourceAuthority = None
     self._sourceNotifications = []  # array of notification addresses
예제 #7
0
    def __init__(self, initType, *args):
        super(UDPTransport, self).__init__()

        templateAddr = None
        if isinstance(initType, ExternalInterfaceTransportInit):
            # External process that is going to talk "in".  There is
            # no parent, and the child is the systemAdmin.
            capabilities, logDefs, concurrency_context = args
            self._adminAddr = self.getAdminAddr(capabilities)
            self._parentAddr = None
        elif isinstance(initType, UDPEndpoint):
            instanceNum, assignedAddr, self._parentAddr, self._adminAddr = initType.args
            templateAddr = assignedAddr
            # N.B.  myAddress is actually the address we will export
            # for others to talk to us, not the bind address.  The
            # difference is that we bind to '0.0.0.0' (inaddr_any),
            # but that's not a valid address for people to send stuff
            # to us.
        else:
            thesplog('UDPTransport init of type %s unsupported',
                     str(initType),
                     level=logging.ERROR)
        if not templateAddr:
            templateAddr = ActorAddress(UDPv4ActorAddress(None, 0))
        self.socket = socket.socket(*templateAddr.addressDetails.socketArgs)
        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.socket.bind(*templateAddr.addressDetails.bindArgs)
        self.myAddress = ActorAddress(
            UDPv4ActorAddress(*self.socket.getsockname(), external=True))
        self._rcvd = []
        self._checkChildren = False
        self._shutdownSignalled = False
        self._pending_actions = []  # array of (ExpirationTimer, func)
예제 #8
0
    def _schedulePreparedIntent(self, transmitIntent):
        # If there's nothing to send, that's implicit success
        if not transmitIntent.serMsg:
            transmitIntent.result = SendStatus.Sent
            transmitIntent.completionCallback()
            return

        # OK, this can be sent now, so go ahead and get it sent out
        if not self._canSendNow():
            self._aTB_queuedPendingTransmits.insert(0, transmitIntent)
            if len(self._aTB_queuedPendingTransmits) >= MAX_QUEUED_TRANSMITS:
                # Try to drain out local work before accepting more
                # because it looks like we're getting really behind.
                # This is dangerous though, because if other Actors
                # are having the same issue this can create a
                # deadlock.
                thesplog(
                    'Entering tx-only mode to drain excessive queue (%s > %s, drain-to %s)',
                    len(self._aTB_queuedPendingTransmits),
                    MAX_QUEUED_TRANSMITS,
                    QUEUE_TRANSMIT_UNBLOCK_THRESHOLD,
                    level=logging.WARNING)
                while len(self._aTB_queuedPendingTransmits
                          ) > QUEUE_TRANSMIT_UNBLOCK_THRESHOLD:
                    self.run(TransmitOnly, transmitIntent.delay())
                thesplog(
                    'Exited tx-only mode after draining excessive queue (%s)',
                    len(self._aTB_queuedPendingTransmits),
                    level=logging.WARNING)
            return

        self._submitTransmit(transmitIntent)
예제 #9
0
 def _handleReadableIncoming(self, inc):
     try:
         rdata = inc.socket.recv(inc.remainingSize())
         inc.failCount = 0
     except socket.error as e:
         inc.failCount = getattr(inc, 'failCount', 0) + 1
         if e.errno in [errno.EAGAIN, errno.EWOULDBLOCK] and inc.failCount < MAX_CONSECUTIVE_READ_FAILURES:
             inc.backoffPause(True)
             return True
         thesplog('Error reading from socket (#%s); closing: %s', inc.failCount, e)
         return False
     if not rdata:
         # Since this point is only arrived at when select() says
         # the socket is readable, this is an indicator of a closed
         # socket.  Since previous calls didn't detect
         # receivedAllData(), this is an aborted/incomplete
         # reception.  Discard it.
         return False
     inc.addData(rdata)
     if not inc.receivedAllData():
         # Continue running and monitoring this socket
         return True
     try:
         rEnv = ReceiveEnvelope(*inc.data)
     except Exception:
         import traceback
         thesplog('OUCH!  Error deserializing received data: %s', traceback.format_exc())
         inc.socket.sendall(ackDataErrMsg)
         # Continue running, but release this socket
         return False
     inc.socket.sendall(ackMsg)
     self._incomingEnvelopes.append(rEnv)
     # Continue to run, but this socket is releasable
     return False
예제 #10
0
    def _runWithExpiry(self, incomingHandler):
        if incomingHandler == TransmitOnly or \
           isinstance(incomingHandler, TransmitOnly):
            # transmits are not queued/multistage in this transport, no waiting
            return 0

        self._aborting_run = False

        while not self.run_time.expired() and not self._aborting_run:
            sresp, _ign1, _ign2 = select.select([self.socket.fileno()], [], [],
                                                self.run_time.remainingSeconds())
            if [] == sresp:
                if [] == _ign1 and [] == _ign2:
                    # Timeout, give up
                    return None
                thesplog('Waiting for read event, but got %s %s', _ign1, _ign2, level=logging.WARNING)
                continue
            rawmsg, sender = self.socket.recvfrom(65535)
            sendAddr = ActorAddress(UDPv4ActorAddress(*sender, external=True))
            try:
                msg = serializer.loads(rawmsg)
            except Exception as ex:
                continue
            if incomingHandler is None:
                return ReceiveEnvelope(sendAddr, msg)
            if not incomingHandler(ReceiveEnvelope(sendAddr, msg)):
                return  # handler returned False, indicating run() should exit

        return None
예제 #11
0
    def _next_XMIT_1(self, intent):
        intent.socket = socket.socket(*intent.targetAddr.addressDetails.socketArgs)
        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        intent.socket.setblocking(0)
        # Disable Nagle to transmit headers and acks asap; our sends are usually small
        intent.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)

        intent.socket.settimeout(timePeriodSeconds(intent.delay()))
        try:
            intent.socket.connect(*intent.targetAddr.addressDetails.connectArgs)
        except socket.error as err:
            # EINPROGRESS means non-blocking socket connect is in progress...
            if err.errno != errno.EINPROGRESS:
                thesplog('Socket connect failure %s to %s (returning %s)',
                         err, intent.targetAddr, intent.completionCallback,
                         level=logging.WARNING)
                return self._finishIntent(intent,
                                          SendStatus.DeadTarget \
                                          if err.errno == errno.ECONNREFUSED \
                                          else SendStatus.Failed)
            intent.backoffPause(True)
        except Exception as ex:
            thesplog('Unexpected TCP socket connect exception: %s', ex,
                     level=logging.ERROR)
            return self._finishIntent(intent, SendStatus.BadPacket)
        intent.stage = self._XMITStepSendData # When connect completes
        intent.amtSent = 0
        return True
예제 #12
0
def _safeSocketShutdown(sock):
    try:
        sock.shutdown(socket.SHUT_RDWR) # KWQ: all these should be protected!
    except socket.error as ex:
        if ex.errno != errno.ENOTCONN:
            thesplog('Error during shutdown of socket %s: %s', sock, ex)
    sock.close()
예제 #13
0
    def _submitTransmit(self, transmitIntent):
        self._aTB_numPendingTransmits += 1
        transmitIntent.addCallback(self._async_txdone, self._async_txdone)

        thesplog('actualTransmit of %s', transmitIntent.identify(),
                 level=logging.DEBUG)
        self._scheduleTransmitActual(transmitIntent)
예제 #14
0
    def _checkConvention(self):
        if self.isConventionLeader():
            missing = []
            for each in self._conventionMembers:
                if self._conventionMembers[each].registryValid.expired():
                    missing.append(each)
            for each in missing:
                thesplog('%s missed %d checkins (%s); assuming it has died',
                         str(self._conventionMembers[each]),
                         CONVENTION_REGISTRATION_MISS_MAX,
                         str(self._conventionMembers[each].registryValid),
                         level=logging.WARNING, primary=True)
                self._remoteSystemCleanup(self._conventionMembers[each].remoteAddress)
            self._conventionRegistration = ExpiryTime(CONVENTION_REREGISTRATION_PERIOD)
        else:
            # Re-register with the Convention if it's time
            if self._conventionAddress and self._conventionRegistration.expired():
                self.setupConvention()

        for each in self._conventionMembers:
            member = self._conventionMembers[each]
            if member.preRegistered and \
               member.preRegistered.pingValid.expired() and \
               not member.preRegistered.pingPending:
                member.preRegistered.pingPending = True
                member.preRegistered.pingValid = ExpiryTime(CONVENTION_RESTART_PERIOD
                                                            if member.registryValid.expired()
                                                            else CONVENTION_REREGISTRATION_PERIOD)
                self._hysteresisSender.sendWithHysteresis(
                    TransmitIntent(member.remoteAddress, ConventionInvite(),
                                   onSuccess = self._preRegQueryNotPending,
                                   onError = self._preRegQueryNotPending))
예제 #15
0
 def _missed_checkin_remote_cleanup(self, remote_member):
     thesplog('%s missed %d checkins (%s); assuming it has died',
              str(remote_member),
              CONVENTION_REGISTRATION_MISS_MAX,
              str(remote_member.registryValid),
              level=logging.WARNING, primary=True)
     return self._remote_system_cleanup(remote_member.remoteAddress)
예제 #16
0
 def run(self):
     # Main loop for convention management.  Wraps the lower-level
     # transport with a stop at the next needed convention
     # registration period to re-register.
     try:
         while not self.isShuttingDown():
             delay = min(self._conventionRegistration or \
                         ExpiryTime(CONVENTION_RESTART_PERIOD
                                    if self._conventionLost and not self.isConventionLeader() else
                                    CONVENTION_REREGISTRATION_PERIOD),
                         ExpiryTime(None) if self._hysteresisSender.delay.expired() else
                         self._hysteresisSender.delay
             )
             # n.b. delay does not account for soon-to-expire
             # pingValids, but since delay will not be longer than
             # a CONVENTION_REREGISTRATION_PERIOD, the worst case
             # is a doubling of a pingValid period (which should be fine).
             self.transport.run(self.handleIncoming, delay.remaining())
             self._checkConvention()
             self._hysteresisSender.checkSends()
     except Exception as ex:
         import traceback
         thesplog('ActorAdmin uncaught exception: %s', traceback.format_exc(),
                  level=logging.ERROR, exc_info=True)
     thesplog('Admin time to die', level=logging.DEBUG)
예제 #17
0
    def _remote_system_cleanup(self, registrant):
        """Called when a RemoteActorSystem has exited and all associated
           Actors should be marked as exited and the ActorSystem
           removed from Convention membership.  This is also called on
           a First Time connection from the remote to discard any
           previous connection information.

        """
        thesplog('Convention cleanup or deregistration for %s (known? %s)',
                 registrant,
                 bool(self._conventionMembers.find(registrant)),
                 level=logging.INFO)
        rmsgs = [LostRemote(registrant)]
        cmr = self._conventionMembers.find(registrant)
        if not cmr or cmr.preRegOnly:
            return []

        # Send exited notification to conventionNotificationHandler (if any)
        for each in self._conventionNotificationHandlers:
            rmsgs.append(
                TransmitIntent(each,
                               ActorSystemConventionUpdate(cmr.remoteAddress,
                                                           cmr.remoteCapabilities,
                                                           False)))  # errors ignored

        # If the remote ActorSystem shutdown gracefully (i.e. sent
        # a Convention Deregistration) then it should not be
        # necessary to shutdown remote Actors (or notify of their
        # shutdown) because the remote ActorSystem should already
        # have caused this to occur.  However, it won't hurt, and
        # it's necessary if the remote ActorSystem did not exit
        # gracefully.

        for lpa, raa in cmr.hasRemoteActors:
            # ignore errors:
            rmsgs.append(TransmitIntent(lpa, ChildActorExited(raa)))
            # n.b. at present, this means that the parent might
            # get duplicate notifications of ChildActorExited; it
            # is expected that Actors can handle this.

        # Remove remote system from conventionMembers
        if not cmr.preRegistered:
            if registrant == self.conventionLeaderAddr and self._invited:
                self._conventionAddress = None
                # Don't clear invited: once invited, that
                # perpetually indicates this should be only a
                # member and never a leader.
            self._conventionMembers.rmv(registrant)
        else:
            # This conventionMember needs to stay because the
            # current system needs to continue issuing
            # registration pings.  By setting the registryValid
            # expiration to forever, this member won't re-time-out
            # and will therefore be otherwise ignored... until it
            # registers again at which point the membership will
            # be updated with new settings.
            cmr.registryValid = ExpirationTimer(None)
            cmr.preRegOnly = True

        return rmsgs + [HysteresisCancel(registrant)]
예제 #18
0
 def shutdown(self):
     thesplog('ActorSystem shutdown requested.', level=logging.INFO)
     time_to_quit = ExpirationTimer(MAX_SYSTEM_SHUTDOWN_DELAY)
     txwatch = self._tx_to_admin(SystemShutdown())
     for remaining_time in unexpired(time_to_quit):
         response = self._run_transport(remaining_time.remaining())
         if txwatch.failed:
             thesplog('Could not send shutdown request to Admin'
                      '; aborting but not necessarily stopped',
                      level=logging.WARNING)
             return
         if isinstance(response, ReceiveEnvelope):
             if isinstance(response.message, SystemShutdownCompleted):
                 break
             else:
                 thesplog('Expected shutdown completed message, got: %s', response.message,
                          level=logging.WARNING)
         elif isinstance(response, (Thespian__Run_Expired,
                                    Thespian__Run_Terminated,
                                    Thespian__Run_Expired)):
             break
         else:
             thesplog('No response to Admin shutdown request; Actor system not completely shutdown',
                      level=logging.ERROR)
     self.transport.close()
     thesplog('ActorSystem shutdown complete.')
예제 #19
0
    def _schedulePreparedIntent(self, transmitIntent):
        # If there's nothing to send, that's implicit success
        if not transmitIntent.serMsg:
            transmitIntent.result = SendStatus.Sent
            transmitIntent.completionCallback()
            return

        # OK, this can be sent now, so go ahead and get it sent out
        if not self._canSendNow():
            self._aTB_queuedPendingTransmits.insert(0, transmitIntent)
            if len(self._aTB_queuedPendingTransmits) >= MAX_QUEUED_TRANSMITS:
                # Try to drain out local work before accepting more
                # because it looks like we're getting really behind.
                # This is dangerous though, because if other Actors
                # are having the same issue this can create a
                # deadlock.
                thesplog('Entering tx-only mode to drain excessive queue (%s > %s, drain-to %s)',
                         len(self._aTB_queuedPendingTransmits),
                         MAX_QUEUED_TRANSMITS,
                         QUEUE_TRANSMIT_UNBLOCK_THRESHOLD,
                         level = logging.WARNING)
                while len(self._aTB_queuedPendingTransmits) > QUEUE_TRANSMIT_UNBLOCK_THRESHOLD:
                    self.run(TransmitOnly, transmitIntent.delay())
                thesplog('Exited tx-only mode after draining excessive queue (%s)',
                         len(self._aTB_queuedPendingTransmits),
                         level = logging.WARNING)
            return

        self._submitTransmit(transmitIntent)
예제 #20
0
 def _sayGoodbye(self):
     self._cleanupAdmin()
     self._send_intent(TransmitIntent(self._exiting,
                                      SystemShutdownCompleted()))
     thesplog('---- shutdown completed', level=logging.INFO)
     logging.info('---- Actor System shutdown')
     self.shutdown_completed = True
예제 #21
0
 def setup_convention(self, activation=False):
     self._has_been_activated |= activation
     rmsgs = []
     # If not specified in capabilities, don't override any invites
     # that may have been received.
     self._conventionAddress = self._getConventionAddr(self.capabilities) or \
                               self._conventionAddress
     leader_is_gone = (self._conventionMembers.find(self.conventionLeaderAddr) is None) \
                      if self.conventionLeaderAddr else True
     if not self.isConventionLeader() and self.conventionLeaderAddr:
         thesplog('Admin registering with Convention @ %s (%s)',
                  self.conventionLeaderAddr,
                  'first time' if leader_is_gone else 're-registering',
                  level=logging.INFO,
                  primary=True)
         rmsgs.append(
             HysteresisSend(self.conventionLeaderAddr,
                            ConventionRegister(self.myAddress,
                                               self.capabilities,
                                               leader_is_gone),
                            onSuccess=self._setupConventionCBGood,
                            onError=self._setupConventionCBError))
         rmsgs.append(LogAggregator(self.conventionLeaderAddr))
     self._conventionRegistration = ExpirationTimer(
         CONVENTION_REREGISTRATION_PERIOD)
     return rmsgs
    def __init__(self, initType, *args):
        super(MultiprocessQueueTransport, self).__init__()
        if isinstance(initType, ExternalInterfaceTransportInit):
            # External process that's going to talk "in".  There is no
            # parent, and the child is the systemAdmin.
            capabilities, logDefs = args
            self._parentQ = None
            self._adminQ = Queue(MAX_ADMIN_QUEUESIZE)
            self._adminAddr = self.getAdminAddr(capabilities)
            self._myQAddress = ActorAddress(QueueActorAddress('~'))
            self._myInputQ = Queue(MAX_ACTOR_QUEUESIZE)
        elif isinstance(initType, MpQTEndpoint):
            _addrInst, myAddr, myQueue, parentQ, adminQ, adminAddr = initType.args
            self._parentQ = parentQ
            self._adminQ = adminQ
            self._adminAddr = adminAddr
            self._myQAddress = myAddr
            self._myInputQ = myQueue
        else:
            thesplog('MultiprocessQueueTransport init of type %s unsupported!',
                     str(initType),
                     level=logging.ERROR)

        # _queues is a map of direct child ActorAddresses to Queue instance.  Note
        # that there will be multiple keys mapping to the same Queue
        # instance because routing is only either to the Parent or to
        # an immediate Child.
        self._queues = {}

        # _fwdvia represents routing for other than immediate parent
        # or child (there may be multiple target addresses mapping to
        # the same forward address.
        self._fwdvia = {}  # key = targetAddress, value=fwdViaAddress

        self._nextSubInstance = 0
예제 #23
0
def _safeSocketShutdown(sock):
    try:
        sock.shutdown(socket.SHUT_RDWR)  # KWQ: all these should be protected!
    except socket.error as ex:
        if ex.errno != errno.ENOTCONN:
            thesplog('Error during shutdown of socket %s: %s', sock, ex)
    sock.close()
예제 #24
0
    def _createInstance(self):
        aClass = self._actorClass
        try:
            aClass = actualActorClass(
                aClass,
                partial(loadModuleFromHashSource, self._sourceHash,
                        self._sources) if self._sourceHash else None)
            # Now instantiate the identified Actor class object
            actorInst = withPossibleInitArgs(capabilities=self.capabilities,
                                             requirements=self._childReqs,
                                             globalName=self.globalName) \
                                             .create(aClass)
            self._sCBStats.inc('Actor.Instance Created')
        except Exception as ex:
            import traceback
            logging.getLogger(str(self._actorClass)) \
                   .error('Actor %s @ %s instantiation exception',
                          self._actorClass, self.transport.myAddress,
                          exc_info = True)
            thesplog('Actor %s @ %s instantiation exception: %s',
                     self._actorClass,
                     self.transport.myAddress,
                     traceback.format_exc(),
                     level=logging.WARNING,
                     primary=True)
            self._sCBStats.inc('Actor.Instance Create Failed')
            self._sayGoodbye()
            return

        self.actorInst = actorInst
        self.actorInst._myRef = self
예제 #25
0
    def _next_XMIT_1(self, intent):
        intent.socket = socket.socket(
            *intent.targetAddr.addressDetails.socketArgs)
        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        intent.socket.setblocking(0)
        # Disable Nagle to transmit headers and acks asap; our sends are usually small
        intent.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)

        intent.socket.settimeout(timePeriodSeconds(intent.delay()))
        try:
            intent.socket.connect(
                *intent.targetAddr.addressDetails.connectArgs)
        except socket.error as err:
            # EINPROGRESS means non-blocking socket connect is in progress...
            if err.errno != errno.EINPROGRESS:
                thesplog('Socket connect failure %s to %s (returning %s)',
                         err,
                         intent.targetAddr,
                         intent.completionCallback,
                         level=logging.WARNING)
                return self._finishIntent(intent,
                                          SendStatus.DeadTarget \
                                          if err.errno == errno.ECONNREFUSED \
                                          else SendStatus.Failed)
            intent.backoffPause(True)
        except Exception as ex:
            thesplog('Unexpected TCP socket connect exception: %s',
                     ex,
                     level=logging.ERROR)
            return self._finishIntent(intent, SendStatus.BadPacket)
        intent.stage = self._XMITStepSendData  # When connect completes
        intent.amtSent = 0
        return True
예제 #26
0
 def _sayGoodbye(self):
     self._cleanupAdmin()
     self._send_intent(
         TransmitIntent(self._exiting, SystemShutdownCompleted()))
     thesplog('---- shutdown completed', level=logging.INFO)
     logging.info('---- Actor System shutdown')
     self.shutdown_completed = True
예제 #27
0
    def __init__(self, initType, *args):
        super(MultiprocessQueueTransport, self).__init__()
        if isinstance(initType, ExternalInterfaceTransportInit):
            # External process that's going to talk "in".  There is no
            # parent, and the child is the systemAdmin.
            capabilities, logDefs, self._concontext = args
            NewQ = self._concontext.Queue if self._concontext else Queue
            self._adminQ     = NewQ(MAX_ADMIN_QUEUESIZE)
            self._adminAddr  = self.getAdminAddr(capabilities)
            self._myQAddress = ActorAddress(QueueActorAddress('~'))
            self._myInputQ   = NewQ(MAX_ACTOR_QUEUESIZE)
            self._QCore = MultiprocessQueueTCore_External(self._myInputQ, None, self._adminQ, self._adminAddr, self._myQAddress)
        elif isinstance(initType, MpQTEndpoint):
            _addrInst, myAddr, myQueue, parentQ, adminQ, adminAddr, ccon = initType.args
            self._concontext = ccon
            self._adminQ     = adminQ
            self._adminAddr  = adminAddr
            self._myQAddress = myAddr
            self._QCore = MultiprocessQueueTCore_Actor(myQueue, parentQ, adminQ, myAddr, myAddr)
        elif isinstance(initType, ExternalQTransportCopy):
            # External process that's going to talk "in".  There is no
            # parent, and the child is the systemAdmin.
            self._QCore, = args
            self._myQAddress = self._QCore.make_external_clone()
        else:
            thesplog('MultiprocessQueueTransport init of type %s unsupported!', str(initType),
                     level=logging.ERROR)

        self._nextSubInstance = 0
    def __init__(self, initType, *args):
        super(MultiprocessQueueTransport, self).__init__()
        if isinstance(initType, ExternalInterfaceTransportInit):
            # External process that's going to talk "in".  There is no
            # parent, and the child is the systemAdmin.
            capabilities, logDefs, self._concontext = args
            NewQ = self._concontext.Queue if self._concontext else Queue
            self._adminQ = NewQ(MAX_ADMIN_QUEUESIZE)
            self._adminAddr = self.getAdminAddr(capabilities)
            self._myQAddress = ActorAddress(QueueActorAddress('~'))
            self._myInputQ = NewQ(MAX_ACTOR_QUEUESIZE)
            self._QCore = MultiprocessQueueTCore_External(
                self._myInputQ, None, self._adminQ, self._adminAddr,
                self._myQAddress)
        elif isinstance(initType, MpQTEndpoint):
            _addrInst, myAddr, myQueue, parentQ, adminQ, adminAddr, ccon = initType.args
            self._concontext = ccon
            self._adminQ = adminQ
            self._adminAddr = adminAddr
            self._myQAddress = myAddr
            self._QCore = MultiprocessQueueTCore_Actor(myQueue, parentQ,
                                                       adminQ, myAddr, myAddr)
        elif isinstance(initType, ExternalQTransportCopy):
            # External process that's going to talk "in".  There is no
            # parent, and the child is the systemAdmin.
            self._QCore, = args
            self._myQAddress = self._QCore.make_external_clone()
        else:
            thesplog('MultiprocessQueueTransport init of type %s unsupported!',
                     str(initType),
                     level=logging.ERROR)

        self._nextSubInstance = 0
예제 #29
0
    def __init__(self, transport, address, capabilities,
                 logdefs,
                 concurrency_context):
        thesplog('++++ Starting Admin from %s',
                 sys.modules['thespian'].__file__,
                 level=logging.DEBUG)
        super(AdminCore, self).__init__(address, transport)
        self.init_replicator(transport, concurrency_context)
        self.capabilities = capabilities
        self.logdefs = logdefs
        self._pendingChildren = {}  # Use: childLocalAddr instance # : PendingActorEnvelope
        # Things that help us look like an Actor, even though we're not
        self._sourceHash = None
        thesplog('++++ Admin started @ %s / gen %s',
                 self.transport.myAddress, str(ThespianGeneration),
                 level=logging.INFO,
                 primary=True)
        logging.info('++++ Actor System gen %s started, admin @ %s',
                     str(ThespianGeneration), self.transport.myAddress)
        logging.debug('Thespian source: %s', sys.modules['thespian'].__file__)
        self._nannying = AssocList()  # child actorAddress -> parent Address
        self._deadLetterHandler = None
        self._sources = {}  # Index is sourcehash, value PendingSource or ValidSource
        self._sourceAuthority = None
        self._sourceNotifications = []  # array of notification addresses

        # The initialization of the Admin and its logger
        # occurs asynchronously, but since the Admin is using a known
        # address, there is nothing to prevent clients from initiating
        # requests to the Admin before it has had a chance to complete
        # the initialization; the _pre_init_msgs will hold those
        # requests until the initialization has completed.
        self._pre_init_msgs = []
예제 #30
0
 def shutdown_signal_detected(signum, frame):
     thesplog('Actor %s @ %s got shutdown signal: %s',
              name,
              addr,
              signum,
              level=logging.WARNING)
     am.transport.interrupt_wait(signal_shutdown=True)
예제 #31
0
 def setupConvention(self):
     if self.isShuttingDown(): return
     if not self._conventionAddress:
         gCA = getattr(self.transport, 'getConventionAddress',
                       lambda c: None)
         self._conventionAddress = gCA(self.capabilities)
         if self._conventionAddress == self.myAddress:
             self._conventionAddress = None
     if self._conventionAddress:
         thesplog('Admin registering with Convention @ %s (%s)',
                  self._conventionAddress,
                  'first time' if getattr(self, '_conventionLeaderIsGone',
                                          True) else 're-registering',
                  level=logging.INFO,
                  primary=True)
         self._hysteresisSender.sendWithHysteresis(
             TransmitIntent(self._conventionAddress,
                            ConventionRegister(
                                self.myAddress, self.capabilities,
                                getattr(self, '_conventionLeaderIsGone',
                                        True)),
                            onSuccess=self._setupConventionCBGood,
                            onError=self._setupConventionCBError))
         self._conventionRegistration = ExpiryTime(
             CONVENTION_REREGISTRATION_PERIOD)
예제 #32
0
    def _remoteSystemCleanup(self, registrant):
        """Called when a RemoteActorSystem has exited and all associated
           Actors should be marked as exited and the ActorSystem
           removed from Convention membership.
        """
        thesplog('Convention cleanup or deregistration for %s (new? %s)',
                 registrant,
                 registrant.actorAddressString not in self._conventionMembers,
                 level=logging.INFO)
        if registrant.actorAddressString in self._conventionMembers:
            cmr = self._conventionMembers[registrant.actorAddressString]

            # Send exited notification to conventionNotificationHandler (if any)
            if self.isConventionLeader():
                for each in self._conventionNotificationHandlers:
                    self._send_intent(
                        TransmitIntent(
                            each,
                            ActorSystemConventionUpdate(
                                cmr.remoteAddress, cmr.remoteCapabilities,
                                False)))  # errors ignored

            # If the remote ActorSystem shutdown gracefully (i.e. sent
            # a Convention Deregistration) then it should not be
            # necessary to shutdown remote Actors (or notify of their
            # shutdown) because the remote ActorSystem should already
            # have caused this to occur.  However, it won't hurt, and
            # it's necessary if the remote ActorSystem did not exit
            # gracefully.

            for lpa, raa in cmr.hasRemoteActors:
                # ignore errors:
                self._send_intent(TransmitIntent(lpa, ChildActorExited(raa)))
                # n.b. at present, this means that the parent might
                # get duplicate notifications of ChildActorExited; it
                # is expected that Actors can handle this.

            # Remove remote system from conventionMembers
            if not cmr.preRegistered:
                del self._conventionMembers[registrant.actorAddressString]
            else:
                # This conventionMember needs to stay because the
                # current system needs to continue issuing
                # registration pings.  By setting the registryValid
                # expiration to forever, this member won't re-time-out
                # and will therefore be otherwise ignored... until it
                # registers again at which point the membership will
                # be updated with new settings.
                cmr.registryValid = ExpiryTime(None)

        if registrant == self._conventionAddress:
            # Convention Leader has exited.  Do NOT set
            # conventionAddress to None.  It might speed up shutdown
            # of this ActorSystem because it won't try to de-register
            # from the convention leader, but if the convention leader
            # reappears there will be nothing to get this ActorSystem
            # re-registered with the convention.
            self._conventionLeaderIsGone = True

        self._hysteresisSender.cancelSends(registrant)
예제 #33
0
    def prepMessageSend(self, anAddress, msg):
        """Prepares to send the specified message to the specified address,
           returning a tuple of the send-to-address and the (possibly
           updated) message to send.

           The address may be converted from an internal to an
           exportable address.

           If the target address is known as a dead letter box, the
           Admin address is returned instead and the message is
           wrapped in a DeadEnvelope wrapper.

           If the target address is not ready for use, the
           send-to-address portion of the tuple will return a value of
           None.

           If the message should no longer be sent, the message
           portion of the tuple will be returned as
           SendStatus.DeadTarget (because this is *never* a valid
           message to actually send).

           n.b. this method may or may not be called while holding a
           lock.  It is a lookup-only operation, so the lock should
           not be of any consequence.
        """
        tgtaddr = self.exportAddr(anAddress)
        if tgtaddr is None:
            return None, msg
        if tgtaddr in self._deadAddrs:
            if isinstance(msg, (DeadEnvelope, ChildActorExited)):
                thesplog('Discarding %s to %s because the latter is dead.',
                         str(msg), str(tgtaddr))
                return None, SendStatus.DeadTarget
            return self._adminAddr, DeadEnvelope(anAddress, msg)
        return tgtaddr, msg
예제 #34
0
    def prepMessageSend(self, anAddress, msg):
        """Prepares to send the specified message to the specified address,
           returning a tuple of the send-to-address and the (possibly
           updated) message to send.

           The address may be converted from an internal to an
           exportable address.

           If the target address is known as a dead letter box, the
           Admin address is returned instead and the message is
           wrapped in a DeadEnvelope wrapper.

           If the target address is not ready for use, the
           send-to-address portion of the tuple will return a value of
           None.

           If the message should no longer be sent, the message
           portion of the tuple will be returned as
           SendStatus.DeadTarget (because this is *never* a valid
           message to actually send).

        """
        tgtaddr = self.exportAddr(anAddress)
        if tgtaddr is None:
            return None, msg
        if tgtaddr in self._deadAddrs:
            if isinstance(msg, (DeadEnvelope, ChildActorExited)):
                thesplog('Discarding %s to %s because the latter is dead.',
                         str(msg), str(tgtaddr))
                return None, SendStatus.DeadTarget
            return self._adminAddr, DeadEnvelope(anAddress, msg)
        return tgtaddr, msg
예제 #35
0
 def _pendingActorResponse(self, envelope):
     # Have seen it arrive here without errorCode set on the PendingActorResponse...
     if not hasattr(envelope.message, 'errorCode'):
         thesplog('Corrupted Pending Actor Response?: %s (%s)',
                  envelope.message,
                  dir(envelope.message),
                  level=logging.ERROR)
         return True
     if not getattr(envelope.message, 'errorCode', 'Failed'):
         self._sCBStats.inc('Actor.Child.Requested.Success')
         self._pendingActorReady(envelope.message.instanceNum,
                                 envelope.message.actualAddress,
                                 isMyChild=not envelope.message.globalName)
         return True
     # Pending Actor Creation failed, clean up all the stuff associated with the intended Actor
     self._sCBStats.inc('Actor.Child.Requested.Failure')
     thesplog('Pending Actor create for %s failed (%s): %s',
              envelope.message.forActor,
              getattr(envelope.message, 'errorCode', '??'),
              getattr(envelope.message, 'errorStr', '---'))
     logging.getLogger(str(self._actorClass)) \
            .error('Pending Actor create for %s failed (%s): %s',
                   envelope.message.forActor,
                   getattr(envelope.message, 'errorCode', '??'),
                   getattr(envelope.message, 'errorStr', '---'))
     # Cancel any queued transmits for this child.
     self._retryPendingChildOperations(envelope.message.instanceNum, None)
     return True
예제 #36
0
 def change_address_for_transmit(self, oldaddr, newaddr):
     oldidx = self._atd.find(oldaddr)
     if oldidx is None:
         # Have not scheduled any transmits for this (probably new)
         # child yet.
         return
     newidx = self._atd.find(newaddr)
     if newidx is None:
         self._atd.add(newaddr, oldidx)
     elif newidx != oldidx:
         if isinstance(oldaddr.addressDetails, ActorLocalAddress):
             # This can happen if sends are made to createActor
             # results with a globalName before the actual address
             # is known.  Each createActor creates a local address,
             # but all those local addresses map back to the same
             # actual address.
             self._ptl[newidx].extend(self._ptl[oldidx])
             self._atd.add(oldaddr, newidx)
             self._ptl[oldidx] = []  # should not be used anymore
         else:
             thesplog(
                 'Duplicate pending transmit indices'
                 ': %s -> %s, %s -> %s',
                 oldaddr,
                 oldidx,
                 newaddr,
                 newidx,
                 level=logging.ERROR)
예제 #37
0
 def __init__(self, childClass, globalName, transport, sourceHash,
              sourceToLoad, parentAddr, adminAddr, childRequirements,
              currentSystemCapabilities, concurrency_context):
     super(ActorManager, self).__init__(adminAddr, transport)
     self.init_replicator(transport, concurrency_context)
     self._parentAddr = parentAddr
     self._sourceHash = sourceHash
     self._sources = {sourceHash: sourceToLoad}
     # Cache the current system capabilities to use for createActor
     # attempts.
     self.capabilities = currentSystemCapabilities
     self._actorClass = childClass  # nb. this may be a string, and sourceHash is not loaded yet
     self._childReqs = childRequirements
     self.actorInst = None
     self.globalName = globalName
     self._srcNotifyEnabled = False
     atexit.register(self._shutdownActor)
     thesplog('Starting Actor %s at %s (parent %s, admin %s, srcHash %s)',
              childClass,
              self.transport.myAddress,
              self._parentAddr,
              adminAddr,
              self._sourceHash,
              level=logging.INFO,
              primary=True)
    def __init__(self, initType, *args):
        super(MultiprocessQueueTransport, self).__init__()
        if isinstance(initType, ExternalInterfaceTransportInit):
            # External process that's going to talk "in".  There is no
            # parent, and the child is the systemAdmin.
            capabilities, logDefs = args
            self._parentQ         = None
            self._adminQ          = Queue(MAX_ADMIN_QUEUESIZE)
            self._adminAddr       = self.getAdminAddr(capabilities)
            self._myQAddress      = ActorAddress(QueueActorAddress('~'))
            self._myInputQ        = Queue(MAX_ACTOR_QUEUESIZE)
        elif isinstance(initType, MpQTEndpoint):
            _addrInst, myAddr, myQueue, parentQ, adminQ, adminAddr = initType.args
            self._parentQ    = parentQ
            self._adminQ     = adminQ
            self._adminAddr  = adminAddr
            self._myQAddress = myAddr
            self._myInputQ   = myQueue
        else:
            thesplog('MultiprocessQueueTransport init of type %s unsupported!', str(initType),
                     level=logging.ERROR)

        # _queues is a map of direct child ActorAddresses to Queue instance.  Note
        # that there will be multiple keys mapping to the same Queue
        # instance because routing is only either to the Parent or to
        # an immediate Child.
        self._queues = {}

        # _fwdvia represents routing for other than immediate parent
        # or child (there may be multiple target addresses mapping to
        # the same forward address.
        self._fwdvia = {} # key = targetAddress, value=fwdViaAddress

        self._nextSubInstance = 0
예제 #39
0
    def _pendingActorReady(self, childInstance, actualAddress):
        if childInstance not in self._pendingChildren:
            thesplog(
                'Pending actor is ready at %s for UNKNOWN %s'
                '; sending child a shutdown',
                actualAddress,
                childInstance,
                level=logging.WARNING)
            self._send_intent(
                TransmitIntent(actualAddress,
                               ActorExitRequest(recursive=True)))
            return

        requestEnvelope = self._pendingChildren[childInstance]
        del self._pendingChildren[childInstance]
        if requestEnvelope.message.globalName or \
           not requestEnvelope.message.forActor:
            # The Admin is the responsible Parent for these children
            self._registerChild(actualAddress)
        else:
            # Anything the Admin was requested to create is a adoptive
            # child and should be killed when the Admin exits.
            self._registerChild(actualAddress)

        if requestEnvelope.message.forActor:
            # Proxy-parenting; remember the real parent
            self._nannying.add(actualAddress, requestEnvelope.message.forActor)
        self._addrManager.associateUseableAddress(self.myAddress,
                                                  childInstance, actualAddress)
        # n.b. childInstance is for this Admin, but caller's
        # childInstance is in original request
        self._sendPendingActorResponse(requestEnvelope, actualAddress)
        self._retryPendingChildOperations(childInstance, actualAddress)
    def _scheduleTransmitActual(self, transmitIntent):
        try:
            if transmitIntent.targetAddr == self.myAddress:
                if transmitIntent.message:
                    self._myInputQ.put( (self._myQAddress, transmitIntent.serMsg), True,
                                        timePeriodSeconds(transmitIntent.delay()))
            else:
                tgtQ = self._queues.find(transmitIntent.targetAddr)
                if tgtQ:
                    tgtQ.put((self._myQAddress, transmitIntent.serMsg), True,
                             timePeriodSeconds(transmitIntent.delay()))
                else:
                    # None means sent by parent, so don't send BACK to parent if unknown
                    topOrFromBelow = self._myQAddress if self._parentQ else None
                    (self._parentQ or self._adminQ).put(
                        (topOrFromBelow, transmitIntent.serMsg),
                        True,
                        timePeriodSeconds(transmitIntent.delay()))

            transmitIntent.tx_done(SendStatus.Sent)
            return
        except Q.Full:
            pass
        transmitIntent.tx_done(SendStatus.DeadTarget if not isinstance(
            transmitIntent._message,
            (ChildActorExited, ActorExitRequest)) else SendStatus.Failed)
        thesplog('Q.Full %s to %s result %s', transmitIntent._message, transmitIntent.targetAddr, transmitIntent.result)
예제 #41
0
    def __init__(self, initType, *args):
        super(UDPTransport, self).__init__()

        templateAddr = None
        if isinstance(initType, ExternalInterfaceTransportInit):
            # External process that is going to talk "in".  There is
            # no parent, and the child is the systemAdmin.
            capabilities, logDefs, concurrency_context = args
            self._adminAddr       = self.getAdminAddr(capabilities)
            self._parentAddr      = None
        elif isinstance(initType, UDPEndpoint):
            instanceNum, assignedAddr, self._parentAddr, self._adminAddr = initType.args
            templateAddr = assignedAddr
            # N.B.  myAddress is actually the address we will export
            # for others to talk to us, not the bind address.  The
            # difference is that we bind to '0.0.0.0' (inaddr_any),
            # but that's not a valid address for people to send stuff
            # to us.
        else:
            thesplog('UDPTransport init of type %s unsupported', str(initType), level=logging.ERROR)
        if not templateAddr:
            templateAddr = ActorAddress(UDPv4ActorAddress(None, 0))
        self.socket = socket.socket(*templateAddr.addressDetails.socketArgs)
        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.socket.bind(*templateAddr.addressDetails.bindArgs)
        self.myAddress = ActorAddress(UDPv4ActorAddress(*self.socket.getsockname(),
                                                        external=True))
        self._rcvd = []
        self._checkChildren = False
        self._shutdownSignalled = False
        self._pending_actions = [] # array of (ExpirationTimer, func)
예제 #42
0
    def _createInstance(self):
        aClass = self._actorClass
        try:
            aClass = actualActorClass(
                aClass,
                partial(loadModuleFromHashSource, self._sourceHash,
                        self._sources) if self._sourceHash else None)
            # Now instantiate the identified Actor class object
            actorInst = aClass()
            self._sCBStats.inc('Actor.Instance Created')
        except Exception as ex:
            import traceback
            logging.getLogger(str(self._actorClass)) \
                   .error('Actor %s @ %s instantiation exception',
                          self._actorClass, self.transport.myAddress,
                          exc_info = True)
            thesplog('Actor %s @ %s instantiation exception: %s',
                     self._actorClass,
                     self.transport.myAddress,
                     traceback.format_exc(),
                     level=logging.ERROR,
                     primary=True)
            self._sCBStats.inc('Actor.Instance Create Failed')
            self._sayGoodbye()
            return

        self.actorInst = actorInst
        self.actorInst._myRef = self
예제 #43
0
    def _remote_system_cleanup(self, registrant):
        """Called when a RemoteActorSystem has exited and all associated
           Actors should be marked as exited and the ActorSystem
           removed from Convention membership.  This is also called on
           a First Time connection from the remote to discard any
           previous connection information.

        """
        thesplog('Convention cleanup or deregistration for %s (known? %s)',
                 registrant,
                 bool(self._conventionMembers.find(registrant)),
                 level=logging.INFO)
        rmsgs = [LostRemote(registrant)]
        cmr = self._conventionMembers.find(registrant)
        if not cmr or cmr.preRegOnly:
            return []

        # Send exited notification to conventionNotificationHandler (if any)
        for each in self._conventionNotificationHandlers:
            rmsgs.append(
                TransmitIntent(each,
                               ActorSystemConventionUpdate(cmr.remoteAddress,
                                                           cmr.remoteCapabilities,
                                                           False)))  # errors ignored

        # If the remote ActorSystem shutdown gracefully (i.e. sent
        # a Convention Deregistration) then it should not be
        # necessary to shutdown remote Actors (or notify of their
        # shutdown) because the remote ActorSystem should already
        # have caused this to occur.  However, it won't hurt, and
        # it's necessary if the remote ActorSystem did not exit
        # gracefully.

        for lpa, raa in cmr.hasRemoteActors:
            # ignore errors:
            rmsgs.append(TransmitIntent(lpa, ChildActorExited(raa)))
            # n.b. at present, this means that the parent might
            # get duplicate notifications of ChildActorExited; it
            # is expected that Actors can handle this.

        # Remove remote system from conventionMembers
        if not cmr.preRegistered:
            if registrant == self.conventionLeaderAddr and self._invited:
                self._conventionAddress = None
                # Don't clear invited: once invited, that
                # perpetually indicates this should be only a
                # member and never a leader.
            self._conventionMembers.rmv(registrant)
        else:
            # This conventionMember needs to stay because the
            # current system needs to continue issuing
            # registration pings.  By setting the registryValid
            # expiration to forever, this member won't re-time-out
            # and will therefore be otherwise ignored... until it
            # registers again at which point the membership will
            # be updated with new settings.
            cmr.registryValid = ExpirationTimer(None)
            cmr.preRegOnly = True

        return rmsgs + [HysteresisCancel(registrant)]
예제 #44
0
    def _pendingActorReady(self, childInstance, actualAddress):
        if childInstance not in self._pendingChildren:
            thesplog('Pending actor is ready at %s for %s but latter is unknown'
                     '; sending child a shutdown',
                     actualAddress, childInstance, level=logging.WARNING)
            self._send_intent(
                TransmitIntent(actualAddress, ActorExitRequest(recursive=True)))
            return

        requestEnvelope = self._pendingChildren[childInstance]
        del self._pendingChildren[childInstance]
        if requestEnvelope.message.globalName or \
           not requestEnvelope.message.forActor:
            # The Admin is the responsible Parent for these children
            self._registerChild(actualAddress)
        else:
            # Anything the Admin was requested to create is a adoptive
            # child and should be killed when the Admin exits.
            self._registerChild(actualAddress)

        if requestEnvelope.message.forActor:
            # Proxy-parenting; remember the real parent
            self._nannying[actualAddress] = requestEnvelope.message.forActor
        self._addrManager.associateUseableAddress(self.myAddress, childInstance, actualAddress)
        # n.b. childInstance is for this Admin, but caller's childInstance is in original request
        self._sendPendingActorResponse(requestEnvelope, actualAddress)
        self._retryPendingChildOperations(childInstance, actualAddress)
예제 #45
0
    def _createInstance(self):
        aClass = self._actorClass
        try:
            aClass = actualActorClass(aClass,
                                      partial(loadModuleFromHashSource,
                                              self._sourceHash,
                                              self._sources)
                                      if self._sourceHash else None)
            # Now instantiate the identified Actor class object
            actorInst = aClass()
            self._sCBStats.inc('Actor.Instance Created')
        except Exception as ex:
            import traceback
            logging.getLogger(str(self._actorClass)) \
                   .error('Actor %s @ %s instantiation exception',
                          self._actorClass, self.transport.myAddress,
                          exc_info = True)
            thesplog('Actor %s @ %s instantiation exception: %s', self._actorClass,
                     self.transport.myAddress, traceback.format_exc(),
                     level=logging.ERROR, primary=True)
            self._sCBStats.inc('Actor.Instance Create Failed')
            self._sayGoodbye()
            return

        self.actorInst = actorInst
        self.actorInst._myRef = self
예제 #46
0
    def run(self):
        # Main loop for convention management.  Wraps the lower-level
        # transport with a stop at the next needed convention
        # registration period to re-register.
        transport_continue = True
        try:
            while not getattr(self, 'shutdown_completed', False) and \
                  not isinstance(transport_continue, Thespian__Run_Terminated):
                ct = currentTime()
                delay = min(
                    self._cstate.convention_inattention_delay(ct),
                    ExpirationTimer(None).view(ct)
                    if self._hysteresisSender.delay.expired() else
                    self._hysteresisSender.delay)
                # n.b. delay does not account for soon-to-expire
                # pingValids, but since delay will not be longer than
                # a CONVENTION_REREGISTRATION_PERIOD, the worst case
                # is a doubling of a pingValid period (which should be fine).
                transport_continue = self.transport.run(
                    self.handleIncoming, delay.remaining())

                # Check Convention status based on the elapsed time
                self._performIO(self._cstate.check_convention())

                self._hysteresisSender.checkSends()
                self._remove_expired_sources()

        except Exception as ex:
            import traceback
            thesplog('ActorAdmin uncaught exception: %s',
                     traceback.format_exc(),
                     level=logging.ERROR,
                     exc_info=True)
        thesplog('Admin time to die', level=logging.DEBUG)
예제 #47
0
    def h_SystemShutdown(self, envelope):
        self._exiting = envelope.sender
        thesplog('---- shutdown initiated by %s', envelope.sender, level=logging.DEBUG)

        # Send failure notices and clear out any pending children.  If
        # any pending child ready notifications are received after
        # this, they will automatically be sent an ActorExitRequest.
        for each in self._pendingChildren:
            pendingReq = self._pendingChildren[each]
            self._send_intent(
                TransmitIntent(
                    pendingReq.sender,
                    PendingActorResponse(
                        pendingReq.message.forActor,
                        pendingReq.message.instanceNum,
                        pendingReq.message.globalName,
                        errorCode = PendingActorResponse.ERROR_ActorSystem_Shutting_Down)))
        self._pendingChildren = []

        if not self.childAddresses:  # no children?
            self._sayGoodbye()
            self.transport.abort_run(drain=True)
            return True

        # Now shutdown any direct children
        self._killLocalActors()

        # Once children confirm their exits the callback will shutdown the Admin.
        return True
예제 #48
0
 def shutdown(self):
     thesplog('ActorSystem shutdown requested.', level=logging.INFO)
     time_to_quit = ExpiryTime(MAX_SYSTEM_SHUTDOWN_DELAY)
     self.transport.scheduleTransmit(
         None,
         TransmitIntent(self.adminAddr,
                        SystemShutdown(),
                        onError=self._shutdownSendFailed))
     while not time_to_quit.expired():
         response = self.transport.run(None, time_to_quit.remaining())
         if getattr(self, '_TASF', False):
             thesplog(
                 'Could not send shutdown request to Admin'
                 '; aborting but not necessarily stopped',
                 level=logging.WARNING)
             return
         if response:
             if isinstance(response.message, SystemShutdownCompleted):
                 break
             else:
                 thesplog('Expected shutdown completed message, got: %s',
                          response.message,
                          level=logging.WARNING)
         else:
             thesplog(
                 'No response to Admin shutdown request; Actor system not completely shutdown',
                 level=logging.ERROR)
     thesplog('ActorSystem shutdown complete.')
예제 #49
0
 def exec_module(self, module):
     moduleName = module.__name__
     hashRoot = self.finder.hashRoot() + '.'
     if moduleName.startswith(hashRoot):
         moduleName = moduleName[len(hashRoot):]
     if self.isModuleDir:
         name = ospath.join(*tuple(moduleName.split('.') + ['__init__.py']))
     elif '.' in moduleName:
         name = ospath.join(*tuple(moduleName.split('.'))) + '.py'
     else:
         name = moduleName + '.py'
     codeproc = lambda s: fix_imports(s, name, hashRoot,
                                      self.finder.getZipTopLevelNames())
     try:
         # Ensure the file ends in a carriage-return.  The path
         # importer does this automatically and no trailing
         # whitespace results in SyntaxError or IndentError
         # exceptions.  In addition, using "universal newlines"
         # mode to read the file is not always effective
         # (e.g. ntlm.HTTPNtlmAuthHandler.py, so explicitly ensure
         # the proper line endings for the compiler.
         if sys.version_info >= (3, 0):
             converter = lambda s: codeproc(s + b'\n')
         else:
             converter = lambda s: codeproc(s.replace('\r\n', '\n') + '\n')
         code = self.finder.withZipElementSource(name, converter)
         do_exec(code, module.__dict__)
     except Exception as ex:
         thesplog('sourceload realization failure: %s',
                  ex,
                  level=logging.ERROR)
         #return None
         raise
예제 #50
0
 def shutdown(self):
     thesplog('ActorSystem shutdown requested.', level=logging.INFO)
     time_to_quit = ExpirationTimer(MAX_SYSTEM_SHUTDOWN_DELAY)
     txwatch = self._tx_to_admin(SystemShutdown())
     while not time_to_quit.expired():
         response = self._run_transport(time_to_quit.remaining())
         if txwatch.failed:
             thesplog('Could not send shutdown request to Admin'
                      '; aborting but not necessarily stopped',
                      level=logging.WARNING)
             return
         if isinstance(response, ReceiveEnvelope):
             if isinstance(response.message, SystemShutdownCompleted):
                 break
             else:
                 thesplog('Expected shutdown completed message, got: %s', response.message,
                          level=logging.WARNING)
         elif isinstance(response, (Thespian__Run_Expired,
                                    Thespian__Run_Terminated,
                                    Thespian__Run_Expired)):
             break
         else:
             thesplog('No response to Admin shutdown request; Actor system not completely shutdown',
                      level=logging.ERROR)
     self.transport.close()
     thesplog('ActorSystem shutdown complete.')
예제 #51
0
    def _checkNextTransmit(self, result, completedIntent):
        # This is the callback for (all) TransmitIntents that will
        # send the next queued intent for that destination.
        if completedIntent.nextIntent:
            self._send_intent_to_transport(completedIntent.nextIntent)
        else:
            fkey = completedIntent.targetAddr
            if fkey not in self._finalTransmitPending:
                fkey = self._addrManager.sendToAddress(completedIntent.targetAddr)
                if fkey not in self._finalTransmitPending:
                    if isinstance(completedIntent.message, DeadEnvelope):
                        fkey = completedIntent.message.deadAddress
                        if fkey not in self._finalTransmitPending:
                            fkey = self._addrManager.sendToAddress(fkey)

            if fkey in self._finalTransmitPending:
                if self._finalTransmitPending[fkey] != completedIntent:
                    thesplog('Completed final intent %s does not match recorded final intent: %s',
                             completedIntent.identify(),
                             self._finalTransmitPending[fkey].identify(),
                             level=logging.WARNING)
                del self._finalTransmitPending[fkey]
            else:
                thesplog('Completed Transmit Intent %s for unrecorded destination %s / %s in %s',
                         completedIntent.identify(),
                         str(self._addrManager.sendToAddress(completedIntent.targetAddr)),
                         fkey,
                         str(map(str,self._finalTransmitPending.keys())),
                         level=logging.WARNING)
                self._sCBStats.inc('Action.Message Send.Unknown Completion')
                return
예제 #52
0
    def run(self):
        # Main loop for convention management.  Wraps the lower-level
        # transport with a stop at the next needed convention
        # registration period to re-register.
        transport_continue = True
        try:
            while not getattr(self, 'shutdown_completed', False) and \
                  not isinstance(transport_continue, Thespian__Run_Terminated):
                delay = min(self._cstate.convention_inattention_delay(),
                            ExpirationTimer(None) if self._hysteresisSender.delay.expired() else
                            self._hysteresisSender.delay
                )
                # n.b. delay does not account for soon-to-expire
                # pingValids, but since delay will not be longer than
                # a CONVENTION_REREGISTRATION_PERIOD, the worst case
                # is a doubling of a pingValid period (which should be fine).
                transport_continue = self.transport.run(self.handleIncoming,
                                                        delay.remaining())

                # Check Convention status based on the elapsed time
                self._performIO(self._cstate.check_convention())

                self._hysteresisSender.checkSends()
                self._remove_expired_sources()

        except Exception as ex:
            import traceback
            thesplog('ActorAdmin uncaught exception: %s', traceback.format_exc(),
                     level=logging.ERROR, exc_info=True)
        thesplog('Admin time to die', level=logging.DEBUG)
예제 #53
0
    def h_SystemShutdown(self, envelope):
        self._exiting = envelope.sender
        thesplog('---- shutdown initiated by %s',
                 envelope.sender,
                 level=logging.DEBUG)

        # Send failure notices and clear out any pending children.  If
        # any pending child ready notifications are received after
        # this, they will automatically be sent an ActorExitRequest.
        for each in self._pendingChildren:
            pendingReq = self._pendingChildren[each]
            self._send_intent(
                TransmitIntent(
                    pendingReq.sender,
                    PendingActorResponse(pendingReq.message.forActor,
                                         pendingReq.message.instanceNum,
                                         pendingReq.message.globalName,
                                         errorCode=PendingActorResponse.
                                         ERROR_ActorSystem_Shutting_Down)))
        self._pendingChildren = []

        if not self.childAddresses:  # no children?
            self._sayGoodbye()
            self.transport.abort_run(drain=True)
            return True

        # Now shutdown any direct children
        self._killLocalActors()

        # Callback will shutdown the Admin Once the children confirm
        # their exits.
        return True
예제 #54
0
 def _missed_checkin_remote_cleanup(self, remote_member):
     thesplog('%s missed %d checkins (%s); assuming it has died',
              str(remote_member),
              CONVENTION_REGISTRATION_MISS_MAX,
              str(remote_member.registryValid),
              level=logging.WARNING, primary=True)
     return self._remote_system_cleanup(remote_member.remoteAddress)
예제 #55
0
 def run(self):
     if self.actorInst is None: self._createInstance()
     if self.actorInst:
         try:
             while True:
                 r = self.transport.run(self.handleMessages)
                 if isinstance(r, Thespian__UpdateWork):
                     self._send_intent(TransmitIntent(
                         self.myAddress, r))  # tickle the transmit queues
                     continue
                 # Expects that on completion of self.transport.run
                 # that the Actor is done processing and that it has
                 # been shutdown gracefully.
                 break
         except Exception as ex:
             # This is usually an internal problem, since the
             # request handling itself catches any exceptions from
             # the Actor itself.
             import traceback
             thesplog('Actor %s @ %s transport run exception: %s',
                      self._actorClass,
                      self.transport.myAddress,
                      traceback.format_exc(),
                      level=logging.ERROR,
                      exc_info=True)
             self._shutdownActor(True)
             self.drainTransmits()
     else:
         self.drainTransmits()
     thesplog('Run %s done', self._actorClass, level=logging.DEBUG)
예제 #56
0
    def _runWithExpiry(self, incomingHandler):
        if incomingHandler == TransmitOnly or \
           isinstance(incomingHandler, TransmitOnly):
            # transmits are not queued/multistage in this transport, no waiting
            return 0

        self._aborting_run = False

        while not self.run_time.expired() and not self._aborting_run:
            if self._rcvd:
                rcvdEnv = self._rcvd.pop()
            else:
                next_action_timeout = self.check_pending_actions()
                try:
                    sresp, _ign1, _ign2 = select.select([self.socket.fileno()], [], [],
                                                        min(self.run_time, next_action_timeout)
                                                        .remainingSeconds())
                except select.error as se:
                    import errno
                    if se.args[0] != errno.EINTR:
                        thesplog('Error during select: %s', se)
                        return None
                    continue
                except ValueError:
                    # self.run_time can expire between the while test
                    # and the use in the select statement.
                    continue

                if [] == sresp:
                    if [] == _ign1 and [] == _ign2:
                        # Timeout, give up
                        return None
                    thesplog('Waiting for read event, but got %s %s', _ign1, _ign2, level=logging.WARNING)
                    continue
                rawmsg, sender = self.socket.recvfrom(65535)
                if rawmsg == b'BuMP':
                    sendAddr = self.myAddress
                    if self._checkChildren:
                        self._checkChildren = False
                        msg = ChildMayHaveDied()
                    elif self._shutdownSignalled:
                        self._shutdownSignalled = False
                        msg = ActorExitRequest()
                    else:
                        return Thespian__UpdateWork()
                else:
                    sendAddr = ActorAddress(UDPv4ActorAddress(*sender, external=True))
                    try:
                        msg = serializer.loads(rawmsg)
                    except Exception:
                        continue
                rcvdEnv = ReceiveEnvelope(sendAddr, msg)
            if incomingHandler is None:
                return rcvdEnv
            r = incomingHandler(rcvdEnv)
            if not r:
                return r # handler returned False, indicating run() should exit

        return None
예제 #57
0
    def _remoteSystemCleanup(self, registrant):
        """Called when a RemoteActorSystem has exited and all associated
           Actors should be marked as exited and the ActorSystem
           removed from Convention membership.
        """
        thesplog('Convention cleanup or deregistration for %s (new? %s)',
                 registrant,
                 registrant.actorAddressString not in self._conventionMembers,
                 level=logging.INFO)
        if registrant.actorAddressString in self._conventionMembers:
            cmr = self._conventionMembers[registrant.actorAddressString]

            # Send exited notification to conventionNotificationHandler (if any)
            if self.isConventionLeader():
                for each in self._conventionNotificationHandlers:
                    self._send_intent(
                        TransmitIntent(each,
                                       ActorSystemConventionUpdate(cmr.remoteAddress,
                                                                   cmr.remoteCapabilities,
                                                                   False)))  # errors ignored

            # If the remote ActorSystem shutdown gracefully (i.e. sent
            # a Convention Deregistration) then it should not be
            # necessary to shutdown remote Actors (or notify of their
            # shutdown) because the remote ActorSystem should already
            # have caused this to occur.  However, it won't hurt, and
            # it's necessary if the remote ActorSystem did not exit
            # gracefully.

            for lpa, raa in cmr.hasRemoteActors:
                # ignore errors:
                self._send_intent(TransmitIntent(lpa, ChildActorExited(raa)))
                # n.b. at present, this means that the parent might
                # get duplicate notifications of ChildActorExited; it
                # is expected that Actors can handle this.

            # Remove remote system from conventionMembers
            if not cmr.preRegistered:
                del self._conventionMembers[registrant.actorAddressString]
            else:
                # This conventionMember needs to stay because the
                # current system needs to continue issuing
                # registration pings.  By setting the registryValid
                # expiration to forever, this member won't re-time-out
                # and will therefore be otherwise ignored... until it
                # registers again at which point the membership will
                # be updated with new settings.
                cmr.registryValid = ExpiryTime(None)

        if registrant == self._conventionAddress:
            # Convention Leader has exited.  Do NOT set
            # conventionAddress to None.  It might speed up shutdown
            # of this ActorSystem because it won't try to de-register
            # from the convention leader, but if the convention leader
            # reappears there will be nothing to get this ActorSystem
            # re-registered with the convention.
            self._conventionLeaderIsGone = True

        self._hysteresisSender.cancelSends(registrant)
예제 #58
0
 def run(self):
     try:
         self.transport.run(self.handleIncoming, None)
     except Exception as ex:
         import traceback
         thesplog('ActorAdmin uncaught exception: %s', traceback.format_exc(),
                  level=logging.ERROR, exc_info=True)
     thesplog('Admin time to die', level=logging.DEBUG)
예제 #59
0
    def scheduleTransmit(self, addressManager, transmitIntent):

        """Requests that a transmit be performed.  The message and target
           address must be fully valid at this point; any local
           addresses should throw a CannotPickleAddress exception and
           the caller is responsible for retrying later when those
           addresses are available.

           If addressManager is None then the intent address is
           assumed to be valid but it cannot be updated if it is a
           local address or a dead address.  A value of None is
           normally only used at Admin or Actor startup time when
           confirming the established connection back to the parent,
           at which time the target address should always be valid.

           Any transmit attempts from a thread other than the main
           thread are queued; calls to the underlying transmit layer
           are done only from the context of the main thread.
        """

        if addressManager:
            # Verify the target address is useable
            targetAddr, txmsg = addressManager.prepMessageSend(
                transmitIntent.targetAddr,
                transmitIntent.message)
            try:
                isDead = txmsg == SendStatus.DeadTarget
            except Exception:
                # txmsg may have an __eq__ that caused an exception
                isDead = False
            if isDead:
                # Address Manager has indicated that these messages
                # should never be attempted because the target is
                # dead.  This is *only* for special messages like
                # DeadEnvelope and ChildActorExited which would
                # endlessly recurse or bounce back and forth.  This
                # code indicates here that the transmit was
                # "successful" to allow normal cleanup but to avoid
                # recursive error generation.
                thesplog('Faking dead target transmit result Sent for %s',
                         transmitIntent, level=logging.WARNING)
                transmitIntent.tx_done(SendStatus.Sent)
                return

            if not targetAddr:
                raise CannotPickleAddress(transmitIntent.targetAddr)

            # In case the prep made some changes...
            transmitIntent.changeTargetAddr(targetAddr)
            transmitIntent.changeMessage(txmsg)

        # Verify that the message can be serialized.  This may throw
        # an exception, which will cause the caller to store this
        # intent and retry it at some future point (the code up to and
        # including this serialization should be idempotent).

        transmitIntent.serMsg = self.serializer(transmitIntent)
        self._schedulePreparedIntent(transmitIntent)
예제 #60
0
 def h_EndpointConnected(self, envelope):
     for C in getattr(self, '_child_procs', []):
         if envelope.message.childInstance == C.childNum:
             C.childRealAddr = envelope.sender
             break
     else:
         thesplog('Unknown child process endpoint connected: %s', envelope, level=logging.WARNING)
     self._pendingActorReady(envelope.message.childInstance, envelope.sender)
     return True