コード例 #1
0
 def testNormalTransmitResetMessage(self):
     ti = TransmitIntent('addr', 'msg')
     self.assertEqual(ti.targetAddr, 'addr')
     self.assertEqual(ti.message, 'msg')
     ti.changeMessage('message2')
     self.assertEqual(ti.targetAddr, 'addr')
     self.assertEqual(ti.message, 'message2')
コード例 #2
0
 def testTransmitIntentSetResult(self):
     ti = TransmitIntent('addr', 'msg')
     assert None == ti.result
     ti.result = SendStatus.Sent
     assert ti.result == SendStatus.Sent
     ti.result = SendStatus.Failed
     assert ti.result == SendStatus.Failed
コード例 #3
0
    def testTransmitIntentRetryTimingExceedsLimit(self):
        maxPeriod = timedelta(seconds=90)
        period = timedelta(microseconds=1)
        now = 1.23
        timepad = timedelta(microseconds=10) # avoid float imprecision
        with update_elapsed_time(now, timedelta(0)):
            ti = TransmitIntent('addr', 'msg',
                                maxPeriod=maxPeriod,
                                retryPeriod=period)
            assert not ti.timeToRetry()

        timeoffset = timedelta(0)
        for N in range(MAX_TRANSMIT_RETRIES+1):
            # Indicate "failure" and the need to retry
            with update_elapsed_time(now, timeoffset + timepad):
                assert ti.retry()
            # Wait for the indication that it is time to retry
            time_to_retry = False
            for x in range(90):
                with update_elapsed_time(now, timeoffset + timepad):
                    # Only call timeToRetry once, because it auto-resets
                    time_to_retry = ti.timeToRetry()
                    if time_to_retry: break
                timeoffset += (period + (period / 2))
                # = period * 1.5, but python2 cannot multiply
                # timedelta by fractions.
            assert time_to_retry

        with update_elapsed_time(now, timeoffset + timepad):
            assert not ti.retry()
コード例 #4
0
 def testNormalTransmitResetMessage(self):
     ti = TransmitIntent('addr', 'msg')
     assert ti.targetAddr == 'addr'
     assert ti.message == 'msg'
     ti.changeMessage('message2')
     assert ti.targetAddr == 'addr'
     assert ti.message == 'message2'
コード例 #5
0
 def testTransmitIntentSetResult(self):
     ti = TransmitIntent('addr', 'msg')
     self.assertEqual(None, ti.result)
     ti.result = SendStatus.Sent
     self.assertEqual(ti.result, SendStatus.Sent)
     ti.result = SendStatus.Failed
     self.assertEqual(ti.result, SendStatus.Failed)
コード例 #6
0
 def testTransmitIntentDelay(self):
     maxPeriod = timedelta(milliseconds=90)
     period = timedelta(milliseconds=30)
     ti = TransmitIntent('addr', 'msg', maxPeriod=maxPeriod, retryPeriod=period)
     delay = ti.delay()
     self.assertGreater(delay, timedelta(milliseconds=88))
     self.assertLess(delay, timedelta(milliseconds=91))
コード例 #7
0
 def testTransmitIntentDelay(self):
     maxPeriod = timedelta(milliseconds=90)
     period = timedelta(milliseconds=30)
     ti = TransmitIntent('addr', 'msg',
                         maxPeriod=maxPeriod,
                         retryPeriod=period)
     delay = ti.delay()
     assert delay > timedelta(milliseconds=88)
     assert delay < timedelta(milliseconds=91)
コード例 #8
0
    def testTransmitIntentRetryTimingExceedsLimit(self):
        maxPeriod = timedelta(seconds=90)
        period = timedelta(microseconds=1)
        ti = TransmitIntent('addr', 'msg', maxPeriod=maxPeriod, retryPeriod=period)
        self.assertFalse(ti.timeToRetry())

        for N in range(MAX_TRANSMIT_RETRIES+1):
            # Indicate "failure" and the need to retry
            self.assertTrue(ti.retry())
            # Wait for the indication that it is time to retry
            time_to_retry = False
            for x in range(90):
                # Only call timeToRetry once, because it auto-resets
                time_to_retry = ti.timeToRetry()
                if time_to_retry: break
                sleep(timePeriodSeconds(period) * 1.5)
            self.assertTrue(time_to_retry)

        self.assertFalse(ti.retry())
コード例 #9
0
 def testTransmitIntentCallbackFailureFailed(self):
     ti = TransmitIntent('addr', 'msg')
     ti.result = SendStatus.Failed
     # Ensure no exception thrown
     ti.completionCallback()
     # And again
     ti.completionCallback()
コード例 #10
0
 def testTransmitIntentCallbackFailureFailedWithChangedTargetsAdded(self):
     self.successes = []
     self.failures = []
     ti = TransmitIntent('addr', 'msg', onSuccess = self._success, onError = self._failed)
     ti.result = SendStatus.Failed
     # Ensure no exception thrown
     ti.completionCallback()
     self.assertEqual(self.successes, [])
     self.assertEqual(self.failures, [(SendStatus.Failed, ti)])
     # And again
     ti.addCallback(self._success, self._failed)
     ti.completionCallback()
     self.assertEqual(self.successes, [])
     self.assertEqual(self.failures, [(SendStatus.Failed, ti), (SendStatus.Failed, ti)])
コード例 #11
0
ファイル: adminCore.py プロジェクト: kquick/Thespian
 def _sendPendingActorResponse(self, requestEnvelope, actualAddress,
                               errorCode=None, errorStr=None):
     # actualAddress is None for failure
     if actualAddress is None and errorCode is None:
         raise ValueError('Must specify either actualAddress or errorCode')
     self._send_intent(
         TransmitIntent(
             requestEnvelope.message.forActor or requestEnvelope.sender,
             PendingActorResponse(requestEnvelope.message.forActor,
                                  requestEnvelope.message.instanceNum,
                                  requestEnvelope.message.globalName,
                                  errorCode=errorCode,
                                  errorStr=errorStr,
                                  actualAddress=actualAddress)))
コード例 #12
0
ファイル: test_hysteresis.py プロジェクト: alex-fu/Thespian
 def testTwoSendsIntentTimeoutIgnored(self):
     self.sends = []
     hs = HysteresisDelaySender(self.send,
                                hysteresis_min_period = timedelta(milliseconds=100),
                                hysteresis_max_period = timedelta(milliseconds=110),
                                hysteresis_rate = 2)
     intent1 = TransmitIntent('addr1', 'msg1')
     intent2 = TransmitIntent('addr1', 'msg2', maxPeriod=timedelta(milliseconds=10))
     hs.sendWithHysteresis(intent1)
     hs.sendWithHysteresis(intent2)
     # First was sent immediately, second is delayed
     assert 1 == len(getattr(self, 'sends', []))
     assert intent1 == self.sends[0]
     assert timedelta(seconds=0) != hs.delay.remaining()
     assert timedelta(milliseconds=110) >= hs.delay.remaining()
     assert timedelta(milliseconds=95) < hs.delay.remaining()
     print('Remaining seconds: %s (%s)'%(hs.delay.remainingSeconds(),
                                         type(hs.delay.remainingSeconds())))
     sleep(hs.delay.remainingSeconds())
     hs.checkSends()
     assert 2 == len(getattr(self, 'sends', []))
     assert intent1 == self.sends[0]
     assert intent2 == self.sends[1]
コード例 #13
0
 def _pendingActorReady(self, childInstance, actualAddress):
     if childInstance in self._pendingChildren:
         gName = self._pendingChildren[childInstance].message.globalName
         if gName:
             if gName in self._globalNames:
                 # This is the loser of the race... just kill it.
                 self._send_intent(
                     TransmitIntent(actualAddress,
                                    ActorExitRequest(recursive=True)))
                 actualAddress = self._globalNames[gName]
             else:
                 self._globalNames[gName] = actualAddress
     return super(GlobalNamesAdmin,
                  self)._pendingActorReady(childInstance, actualAddress)
コード例 #14
0
 def testTransmitIntentCallbackSuccessWithChangedTargetsAdded(self):
     self.successes = []
     self.failures = []
     ti = TransmitIntent('addr', 'msg',
                         onSuccess = self._success,
                         onError = self._failed)
     ti.result = SendStatus.Sent
     # Ensure no exception thrown
     ti.completionCallback()
     assert self.successes == [(SendStatus.Sent, ti)]
     assert self.failures == []
     # And again
     ti.addCallback(self._success, self._failed)
     ti.completionCallback()
     assert self.successes == [(SendStatus.Sent, ti), (SendStatus.Sent, ti)]
     assert self.failures == []
コード例 #15
0
    def handle(self, record):
        # Can't pickle traceback objects, so pre-format them.  Sorry,
        # more logging internals here.

        # Originally *all* messages were forwarded from actors to the
        # logging actor, and the filtering/level management was
        # performed by the latter.
        #
        # However, some packages
        # (e.g. sqlalchemy) generate large amounts of debug logging
        # with the expectation that those are normally filtered and
        # hidden.  This could cause excessive amounts of transmits
        # from an actor to the logger, to the point that the TxOnly
        # threshold was exceeded and actor responsiveness was
        # degraded.
        #
        # To avoid that degradation, the level set for this handler is
        # used to filter messages prior to sending them to the logger.
        # The downside is that the logging level cannot be globally
        # changed, although it can be adjusted on a per-actor basis.
        if record.levelno < self.level:
            return

        global logTransport
        if record.exc_info:
            if not record.exc_text:
                excinfo = traceback.format_exception(record.exc_info[0],
                                                     record.exc_info[1],
                                                     record.exc_info[2])
                record.exc_text = '\n'.join(excinfo)
            record.exc_info = None
        record.__dict__['actorAddress'] = str(logTransport.myAddress)
        try:
            msg = record.getMessage()
        except Exception:
            newargs = []
            for arg in record.args:
                try:
                    newargs.append(str(arg))
                except Exception:
                    newargs.append('<un-str-able argument>')
            record.args = newargs
            try:
                msg = record.getMessage()
            except Exception:
                msg = '<un-str-able message>'
        record.msg = msg
        record.args = None
        logTransport.scheduleTransmit(None,
                                      TransmitIntent(self._fwdAddr, record))
コード例 #16
0
 def unloadActorSource(self, sourceHash):
     if sourceHash in self._sources:
         msg = UnloadedSource(self._sources[sourceHash].srcHash,
                              self._sources[sourceHash].srcInfo)
         for each in self._sourceNotifications:
             self._send_intent(TransmitIntent(each, msg))
         del self._sources[sourceHash]
     for pnum, metapath in enumerate(sys.meta_path):
         if getattr(metapath, 'srcHash', None) == sourceHash:
             rmmods = [M for M in sys.modules
                       if M and M.startswith(metapath.hashRoot())]
             for each in rmmods:
                 del sys.modules[each]
             del sys.meta_path[pnum]
             break
コード例 #17
0
ファイル: adminCore.py プロジェクト: nabheet/Thespian
 def h_NotifyOnSourceAvailability(self, envelope):
     address = envelope.message.notificationAddress
     enable = envelope.message.enable
     all_except = [A for A in self._sourceNotifications if A != address]
     if enable:
         self._sourceNotifications = all_except + [address]
         for each in self._sources:
             if self._sources[each].source_valid:
                 self._send_intent(
                     TransmitIntent(
                         address,
                         LoadedSource(self._sources[each].srcHash,
                                      self._sources[each].srcInfo)))
     else:
         self._sourceNotifications = all_except
コード例 #18
0
ファイル: logdirector.py プロジェクト: thomcost/Thespian
 def handle(self, record):
     # Can't pickle traceback objects, so pre-format them.  Sorry,
     # more logging internals here.
     if record.exc_info:
         if not record.exc_text:
             excinfo = traceback.format_exception(record.exc_info[0],
                                                  record.exc_info[1],
                                                  record.exc_info[2])
             record.exc_text = '\n'.join(excinfo)
         record.exc_info = None
     record.__dict__['actorAddress'] = str(self._transport.myAddress)
     msg = record.getMessage()
     record.msg = msg
     record.args = None
     self._transport.scheduleTransmit(None,
                                      TransmitIntent(self._fwdAddr, record))
コード例 #19
0
ファイル: adminCore.py プロジェクト: jfasenfest/Thespian
 def h_NotifyOnSourceAvailability(self, envelope):
     address = envelope.message.notificationAddress
     enable = envelope.message.enable
     all_except = list(
         filter(lambda a: a != address, self._sourceNotifications))
     if enable:
         self._sourceNotifications = all_except + [address]
         for each in self._sources:
             if hasattr(self._sources[each], 'srcHash'):
                 self._send_intent(
                     TransmitIntent(
                         address,
                         LoadedSource(self._sources[each].srcHash,
                                      self._sources[each].srcInfo)))
     else:
         self._sourceNotifications = all_except
コード例 #20
0
ファイル: convention.py プロジェクト: waqaraqeel/Thespian
 def _performIO(self, iolist):
     for msg in iolist:
         if isinstance(msg, HysteresisCancel):
             self._hysteresisSender.cancelSends(msg.cancel_addr)
         elif isinstance(msg, HysteresisSend):
             #self._send_intent(msg)
             self._hysteresisSender.sendWithHysteresis(msg)
         elif isinstance(msg, LogAggregator):
             if getattr(self, 'asLogger', None):
                 thesplog('Setting log aggregator of %s to %s', self.asLogger, msg.aggregatorAddress)
                 self._send_intent(TransmitIntent(self.asLogger, msg))
         elif isinstance(msg, LostRemote):
             if hasattr(self.transport, 'lostRemote'):
                 self.transport.lostRemote(msg.lost_addr)
         else:
             self._send_intent(msg)
コード例 #21
0
 def h_NotifyOnSystemRegistration(self, envelope):
     if envelope.message.enableNotification:
         newRegistrant = envelope.sender not in self._conventionNotificationHandlers
         if newRegistrant:
             self._conventionNotificationHandlers.add(envelope.sender)
             # Now update the registrant on the current state of all convention members
             for member in self._conventionMembers:
                 self._send_intent(
                     TransmitIntent(
                         envelope.sender,
                         ActorSystemConventionUpdate(
                             member, self._conventionMembers[member].
                             remoteCapabilities, True)))
     else:
         self._conventionNotificationHandlers.discard(envelope.sender)
     return True
コード例 #22
0
    def _loadValidatedActorSource(self, sourceHash, sourceZip, sourceInfo):
        # Validate the source file; this doesn't actually utilize the
        # sourceZip, but it ensures that the sourceZip isn't garbage
        # before registering it as active source.
        if sourceHash not in self._sources:
            logging.getLogger('Thespian').warning(
                'Provided validated source with no or expired request'
                ', hash %s; ignoring.', sourceHash)
            return

        try:
            f = SourceHashFinder(sourceHash, lambda v: v, sourceZip)
            namelist = f.getZipNames()
            logging.getLogger('Thespian').info(
                'Validated source hash %s - %s, %s modules (%s)',
                sourceHash, sourceInfo, len(namelist),
                ', '.join(namelist if len(namelist) < 10 else
                          namelist[:9] + ['...']))
        except Exception as ex:
            logging.getLogger('Thespian')\
                   .error('Validated source (hash %s) is corrupted: %s',
                          sourceHash, ex)
            return

        if self._sources[sourceHash].source_valid:
            # If a duplicate source load request is made while the
            # first is still being validated by the Source Authority,
            # another request will be sent to the Source Authority and
            # the latter response will be a duplicate here and can
            # simply be dropped.
            return

        # Store this registered source
        pending_actors = self._sources[sourceHash].pending_actors

        self._sources[sourceHash] = ValidSource(sourceHash,
                                                self._sources[sourceHash].orig_data,
                                                sourceZip,
                                                str(sourceInfo))

        for each in pending_actors:
            self.h_PendingActor(each)

        msg = LoadedSource(self._sources[sourceHash].srcHash,
                           self._sources[sourceHash].srcInfo)
        for each in self._sourceNotifications:
            self._send_intent(TransmitIntent(each, msg))
コード例 #23
0
ファイル: convention.py プロジェクト: waqaraqeel/Thespian
 def got_convention_deregister(self, deregmsg):
     self._sCBStats.inc('Admin Handle Convention De-registration')
     remoteAdmin = deregmsg.adminAddress
     if remoteAdmin == self.myAddress:
         # Either remote failed getting an external address and is
         # using 127.0.0.1 or else this is a malicious attempt to
         # make us talk to ourselves.  Ignore it.
         thesplog('Convention deregistration from %s is an invalid address; ignoring.',
                  remoteAdmin,
                  level=logging.WARNING)
     rmsgs = []
     if getattr(deregmsg, 'preRegistered', False): # see definition for getattr use
         existing = self._conventionMembers.find(remoteAdmin)
         if existing:
             existing.preRegistered = None
             rmsgs.append(TransmitIntent(remoteAdmin, ConventionDeRegister(self.myAddress)))
     return rmsgs + self._remote_system_cleanup(remoteAdmin)
コード例 #24
0
 def testTransmitIntentCallbackFailureFailedWithTarget(self):
     self.successes = []
     self.failures = []
     ti = TransmitIntent('addr', 'msg',
                         onSuccess = self._success,
                         onError = self._failed)
     ti.result = SendStatus.Failed
     # Ensure no exception thrown
     ti.completionCallback()
     assert self.successes == []
     assert self.failures == [(SendStatus.Failed, ti)]
     # And again
     ti.completionCallback()
     assert self.successes == []
     assert self.failures == [(SendStatus.Failed, ti)]
コード例 #25
0
ファイル: convention.py プロジェクト: alex-fu/Thespian
 def got_system_shutdown(self):
     gen_ops = lambda addr: [
         HysteresisCancel(addr),
         TransmitIntent(addr, ConventionDeRegister(self.myAddress)),
     ]
     if self.conventionLeaderAddr and \
        self.conventionLeaderAddr != self.myAddress:
         thesplog('Admin de-registering with Convention @ %s',
                  str(self.conventionLeaderAddr),
                  level=logging.INFO,
                  primary=True)
         return gen_ops(self.conventionLeaderAddr)
     return join(
         fmap(gen_ops, [
             M.remoteAddress for M in self._conventionMembers.values()
             if M.remoteAddress != self.myAddress
         ]))
コード例 #26
0
ファイル: convention.py プロジェクト: kquick/Thespian
    def forward_pending_to_remote_system(self, childClass, envelope,
                                         sourceHash, acceptsCaps):
        alreadyTried = getattr(envelope.message, 'alreadyTried', [])
        ct = currentTime()
        if self.myAddress not in alreadyTried:
            # Don't send request back to this actor system: it cannot
            # handle it
            alreadyTried.append(self.myAddress)

        remoteCandidates = [
            K for K in self._conventionMembers.values()
            if not K.registryValid.view(ct).expired()
            and K.remoteAddress != envelope.sender  # source Admin
            and K.remoteAddress not in alreadyTried
            and acceptsCaps(K.remoteCapabilities)
        ]

        if not remoteCandidates:
            if self.isConventionLeader() or not self.conventionLeaderAddr:
                raise NoCompatibleSystemForActor(
                    childClass, 'No known ActorSystems can handle a %s for %s',
                    childClass, envelope.message.forActor)
            # Let the Convention Leader try to find an appropriate ActorSystem
            bestC = self.conventionLeaderAddr
        else:
            # distribute equally amongst candidates
            C = [(K.remoteAddress, len(K.hasRemoteActors))
                 for K in remoteCandidates]
            bestC = foldl(
                lambda best, possible: best
                if best[1] <= possible[1] else possible, C)[0]
            thesplog('Requesting creation of %s%s on remote admin %s',
                     envelope.message.actorClassName,
                     ' (%s)' % sourceHash if sourceHash else '', bestC)
        if bestC in alreadyTried:
            return []  # Have to give up, no-one can handle this

        # Don't send request to this remote again, it has already
        # been tried.  This would also be indicated by that system
        # performing the add of self.myAddress as below, but if
        # there is disagreement between the local and remote
        # addresses, this addition will prevent continual
        # bounceback.
        alreadyTried.append(bestC)
        envelope.message.alreadyTried = alreadyTried
        return [TransmitIntent(bestC, envelope.message)]
コード例 #27
0
ファイル: test_asyncBase.py プロジェクト: thomcost/Thespian
 def test_pendingCallbacksClearQueueAndMoreRunsRunAdditionalQueuedOnMoreCompletions(
         self):
     numExtras = len(self.extraTransmitIds)
     self.test_extraPendingCallbackCompletionsDoNothing()
     # Add more transmits to reach MAX_PENDING_TRANSMITS again
     for _moreExtras in range(numExtras + 3):
         self.testTrans.forTestingCompleteAPendingIntent(SendStatus.Sent)
     expectedCBCount = 2 * (numExtras + 3)
     self.assertEqual(self.successCBcalls, expectedCBCount)
     self.assertEqual(MAX_PENDING_TRANSMITS + numExtras,
                      len(self.testTrans.intents))
     self.assertEqual(
         numExtras,
         len([
             I for I in self.testTrans.intents
             if I.message in self.extraTransmitIds
         ]))
     # Now send more and make sure they are queued
     for count in self.extraTransmitIds:
         self.testTrans.scheduleTransmit(
             None,
             TransmitIntent(ActorAddress(3.5), count, self.successCB,
                            self.failureCB))
     self.assertEqual(self.successCBcalls, expectedCBCount)
     self.assertEqual(MAX_PENDING_TRANSMITS + 2 * numExtras,
                      len(self.testTrans.intents))
     self.assertEqual(
         2 * numExtras,
         len([
             I for I in self.testTrans.intents
             if I.message in self.extraTransmitIds
         ]))
     # And verify that more completions cause the newly Queued to be run
     for _extras in range(numExtras):
         self.testTrans.forTestingCompleteAPendingIntent(SendStatus.Sent)
     expectedCBCount += numExtras
     self.assertEqual(self.successCBcalls, expectedCBCount)
     self.assertEqual(MAX_PENDING_TRANSMITS + numExtras + numExtras,
                      len(self.testTrans.intents))
     self.assertEqual(
         2 * numExtras,
         len([
             I for I in self.testTrans.intents
             if I.message in self.extraTransmitIds
         ]))
コード例 #28
0
 def testTransmitIntentCallbackFailureNotSentWithTarget(self):
     self.successes = []
     self.failures = []
     ti = TransmitIntent('addr',
                         'msg',
                         onSuccess=self._success,
                         onError=self._failed)
     ti.result = SendStatus.NotSent
     # Ensure no exception thrown
     ti.completionCallback()
     self.assertEqual(self.successes, [])
     self.assertEqual(self.failures, [(SendStatus.NotSent, ti)])
     # And again
     ti.completionCallback()
     self.assertEqual(self.successes, [])
     self.assertEqual(self.failures, [(SendStatus.NotSent, ti)])
コード例 #29
0
ファイル: globalNames.py プロジェクト: thomcost/Thespian
    def h_PendingActor(self, envelope):
        gName = envelope.message.globalName
        if not gName:
            return super(GlobalNamesAdmin, self).h_PendingActor(envelope)

        if gName in self._globalNames:
            # Actor already registered with this name... here it is.
            self._send_intent(
                TransmitIntent(
                    envelope.sender,
                    PendingActorResponse(
                        envelope.message.forActor,
                        envelope.message.instanceNum,
                        gName,
                        actualAddress=self._globalNames[gName])))
            return True

        return super(GlobalNamesAdmin, self).h_PendingActor(envelope)
コード例 #30
0
 def testTransmitIntentCallbackFailureNotSentWithTarget(self):
     self.successes = []
     self.failures = []
     ti = TransmitIntent('addr', 'msg', onSuccess = self._success, onError = self._failed)
     ti.result = SendStatus.NotSent
     # Ensure no exception thrown
     ti.completionCallback()
     self.assertEqual(self.successes, [])
     self.assertEqual(self.failures, [(SendStatus.NotSent, ti)])
     # And again
     ti.completionCallback()
     self.assertEqual(self.successes, [])
     self.assertEqual(self.failures, [(SendStatus.NotSent, ti)])
コード例 #31
0
    def testTransmitIntentRetryTimingExceedsLimit(self):
        maxPeriod = timedelta(seconds=90)
        period = timedelta(microseconds=1)
        ti = TransmitIntent('addr',
                            'msg',
                            maxPeriod=maxPeriod,
                            retryPeriod=period)
        self.assertFalse(ti.timeToRetry())

        for N in range(MAX_TRANSMIT_RETRIES + 1):
            self.assertTrue(ti.retry())
            for x in range(90):
                if ti.timeToRetry(): break
                sleep(timePeriodSeconds(period))
            self.assertTrue(ti.timeToRetry())

        self.assertFalse(ti.retry())
コード例 #32
0
ファイル: adminCore.py プロジェクト: jfasenfest/Thespian
 def run(self):
     try:
         while True:
             r = self.transport.run(self.handleIncoming, None)
             if isinstance(r, Thespian__UpdateWork):
                 # tickle the transmit queues
                 self._send_intent(TransmitIntent(self.myAddress, r))
                 continue
             # Expects that on completion of self.transport.run
             # that the Actor is done processing and that it has
             # been shutdown gracefully.
             break
     except Exception:
         import traceback
         thesplog('ActorAdmin uncaught exception: %s',
                  traceback.format_exc(),
                  level=logging.ERROR,
                  exc_info=True)
     thesplog('Admin time to die', level=logging.DEBUG)
コード例 #33
0
    def testTransmitIntentRetryTimingExceedsLimit(self):
        maxPeriod = timedelta(seconds=90)
        period = timedelta(microseconds=1)
        ti = TransmitIntent('addr', 'msg', maxPeriod=maxPeriod, retryPeriod=period)
        self.assertFalse(ti.timeToRetry())

        for N in range(MAX_TRANSMIT_RETRIES+1):
            self.assertTrue(ti.retry())
            for x in range(90):
                if ti.timeToRetry(): break
                sleep(timePeriodSeconds(period))
            self.assertTrue(ti.timeToRetry())

        self.assertFalse(ti.retry())
コード例 #34
0
ファイル: convention.py プロジェクト: kquick/Thespian
    def _get_missing_source_for_hash(self, sourceHash, createActorEnvelope):
        # If this request was forwarded by a remote Admin and the
        # sourceHash is not known locally, request it from the sending
        # remote Admin
        if self._cstate.sentByRemoteAdmin(createActorEnvelope) and \
           self._acceptsRemoteLoadedSourcesFrom(createActorEnvelope):
            self._sources[sourceHash] = PendingSource(sourceHash, None)
            self._sources[sourceHash].pending_actors.append(
                createActorEnvelope)
            self._hysteresisSender.sendWithHysteresis(
                TransmitIntent(
                    createActorEnvelope.sender,
                    SourceHashTransferRequest(sourceHash,
                                              bool(self._sourceAuthority))))
            # sent with hysteresis, so break out to local _run
            return False

        # No remote Admin to send the source, so fail as normal.
        return super(ConventioneerAdmin, self)._get_missing_source_for_hash(
            sourceHash, createActorEnvelope)
コード例 #35
0
ファイル: convention.py プロジェクト: waqaraqeel/Thespian
 def exit_convention(self):
     self.invited = False
     gen_ops = lambda addr: [HysteresisCancel(addr),
                             TransmitIntent(addr,
                                            ConventionDeRegister(self.myAddress)),
     ]
     terminate = lambda a: [ self._remote_system_cleanup(a), gen_ops(a) ][-1]
     if self.conventionLeaderAddr and \
        self.conventionLeaderAddr != self.myAddress:
         thesplog('Admin de-registering with Convention @ %s',
                  str(self.conventionLeaderAddr),
                  level=logging.INFO, primary=True)
         # Cache convention leader address because it might get reset by terminate()
         claddr = self.conventionLeaderAddr
         terminate(self.conventionLeaderAddr)
         return gen_ops(claddr)
     return join(fmap(terminate,
                      [M.remoteAddress
                       for M in self._conventionMembers.values()
                       if M.remoteAddress != self.myAddress]))
コード例 #36
0
 def testTransmitIntentCallbackFailureFailedWithTarget(self):
     self.successes = []
     self.failures = []
     ti = TransmitIntent('addr', 'msg',
                         onSuccess = self._success,
                         onError = self._failed)
     ti.result = SendStatus.Failed
     # Ensure no exception thrown
     ti.completionCallback()
     assert self.successes == []
     assert self.failures == [(SendStatus.Failed, ti)]
     # And again
     ti.completionCallback()
     assert self.successes == []
     assert self.failures == [(SendStatus.Failed, ti)]
コード例 #37
0
ファイル: adminCore.py プロジェクト: jfasenfest/Thespian
 def h_ValidateSource(self, envelope):
     sourceHash = envelope.message.sourceHash
     if not envelope.message.sourceData:
         self.unloadActorSource(sourceHash)
         logging.getLogger('Thespian')\
                .info('Source hash %s unloaded', sourceHash)
         return
     if sourceHash in self._sources:
         logging.getLogger('Thespian')\
                .info('Source hash %s (%s) already loaded', sourceHash,
                      self._sources[sourceHash].srcInfo
                      if isinstance(self._sources[sourceHash], ValidSource)
                      else '<pending>')
         return
     if self._sourceAuthority:
         self._send_intent(
             TransmitIntent(self._sourceAuthority, envelope.message))
         return
     # Any attempt to load sources is ignored if there is no active
     # Source Authority.  This is a security measure to protect the
     # un-protected.
     logging.getLogger('Thespian').warning(
         'No source authority to validate source hash %s', sourceHash)
コード例 #38
0
    def _checkConvention(self):
        if self.isConventionLeader():
            missing = []
            for each in self._conventionMembers:
                if self._conventionMembers[each].registryValid.expired():
                    missing.append(each)
            for each in missing:
                thesplog('%s missed %d checkins (%s); assuming it has died',
                         str(self._conventionMembers[each]),
                         CONVENTION_REGISTRATION_MISS_MAX,
                         str(self._conventionMembers[each].registryValid),
                         level=logging.WARNING,
                         primary=True)
                self._remoteSystemCleanup(
                    self._conventionMembers[each].remoteAddress)
            self._conventionRegistration = ExpiryTime(
                CONVENTION_REREGISTRATION_PERIOD)
        else:
            # Re-register with the Convention if it's time
            if self._conventionAddress and self._conventionRegistration.expired(
            ):
                self.setupConvention()

        for each in self._conventionMembers:
            member = self._conventionMembers[each]
            if member.preRegistered and \
               member.preRegistered.pingValid.expired() and \
               not member.preRegistered.pingPending:
                member.preRegistered.pingPending = True
                member.preRegistered.pingValid = ExpiryTime(
                    CONVENTION_RESTART_PERIOD if member.registryValid.expired(
                    ) else CONVENTION_REREGISTRATION_PERIOD)
                self._hysteresisSender.sendWithHysteresis(
                    TransmitIntent(member.remoteAddress,
                                   ConventionInvite(),
                                   onSuccess=self._preRegQueryNotPending,
                                   onError=self._preRegQueryNotPending))
コード例 #39
0
ファイル: logdirector.py プロジェクト: asevans48/Thespian
def startupASLogger(addrOfStarter, logEndpoint, logDefs, transportClass,
                    aggregatorAddress):
    # Dirty trick here to completely re-initialize logging in this
    # process... something the standard Python logging interface does
    # not allow via the API.  We also do not want to run
    # logging.shutdown() because (a) that does not do enough to reset,
    # and (b) it shuts down handlers, but we want to leave the
    # parent's handlers alone.  Dirty trick here to completely
    # re-initialize logging in this process... something the standard
    # Python logging interface does not allow via the API.
    logging.root = logging.RootLogger(logging.WARNING)
    logging.Logger.root = logging.root
    logging.Logger.manager = logging.Manager(logging.Logger.root)
    if logDefs:
        dictConfig(logDefs)
    else:
        logging.basicConfig()
    # Disable thesplog from within the logging process (by setting the
    # logfile size to zero) to try to avoid recursive logging loops.
    thesplog_control(logging.WARNING, False, 0)
    #logging.info('ActorSystem Logging Initialized')
    transport = transportClass(logEndpoint)
    setProcName('logger', transport.myAddress)
    transport.scheduleTransmit(
        None, TransmitIntent(addrOfStarter, LoggerConnected()))
    fdup = None
    last_exception_time = None
    exception_count = 0
    while True:
        try:
            r = transport.run(None)
            if isinstance(r, Thespian__UpdateWork):
                transport.scheduleTransmit(
                    TransmitIntent(transport.myAddress, r))
                continue
            logrecord = r.message
            if isinstance(logrecord, LoggerExitRequest):
                logging.info('ActorSystem Logging Shutdown')
                return
            elif isinstance(logrecord, LoggerFileDup):
                fdup = getattr(logrecord, 'fname', None)
            elif isinstance(logrecord, LogAggregator):
                aggregatorAddress = logrecord.aggregatorAddress
            elif isinstance(logrecord, logging.LogRecord):
                logging.getLogger(logrecord.name).handle(logrecord)
                if fdup:
                    with open(fdup, 'a') as ldf:
                        ldf.write('%s\n' % str(logrecord))
                if aggregatorAddress and \
                   logrecord.levelno >= logging.WARNING:
                    transport.scheduleTransmit(
                        None, TransmitIntent(aggregatorAddress, logrecord))
            else:
                logging.warn('Unknown message rcvd by logger: %s' %
                             str(logrecord))
        except Exception as ex:
            thesplog('Thespian Logger aborting (#%d) with error %s',
                     exception_count,
                     ex,
                     exc_info=True)
            if last_exception_time is None or \
               last_exception_time.view().expired():
                last_exception_time = ExpirationTimer(timedelta(seconds=1))
                exception_count = 0
            else:
                exception_count += 1
                if exception_count >= MAX_LOGGING_EXCEPTIONS_PER_SECOND:
                    thesplog(
                        'Too many Thespian Logger exceptions (#%d in %s); exiting!',
                        exception_count,
                        timedelta(seconds=1) -
                        last_exception_time.view().remaining())
                    return
コード例 #40
0
 def testTransmitIntentRetry(self):
     ti = TransmitIntent('addr', 'msg')
     for x in range(MAX_TRANSMIT_RETRIES+1):
         assert ti.retry()
     assert not ti.retry()
コード例 #41
0
    def testTransmitIntentRetryTiming(self):
        maxPeriod = timedelta(milliseconds=90)
        period = timedelta(milliseconds=30)
        now = 0.01
        timepad = timedelta(microseconds=10) # avoid float imprecision
        with update_elapsed_time(now, timedelta(0)):
            ti = TransmitIntent('addr', 'msg',
                                maxPeriod=maxPeriod,
                                retryPeriod=period)
            assert not ti.timeToRetry()

        with update_elapsed_time(now, period + timepad):
            assert not ti.timeToRetry()

            assert ti.retry()
            assert not ti.timeToRetry()

        with update_elapsed_time(now, period + period + timepad):
            assert ti.timeToRetry()
            assert ti.retry()
            assert not ti.timeToRetry()

        with update_elapsed_time(now, period * 3 + timepad):
            assert not ti.timeToRetry()  # Each retry increases

        with update_elapsed_time(now, period * 4 + timepad):
            assert ti.timeToRetry()
            assert not ti.retry()  # Exceeds maximum time
コード例 #42
0
 def testNormalTransmitIdentification(self):
     ti = TransmitIntent('addr', 'msg')
     # Just ensure no exceptions are thrown
     self.assertTrue(ti.identify())
コード例 #43
0
    def testTransmitIntentRetryTiming(self):
        maxPeriod = timedelta(milliseconds=90)
        period = timedelta(milliseconds=30)
        ti = TransmitIntent('addr', 'msg', maxPeriod=maxPeriod, retryPeriod=period)
        self.assertFalse(ti.timeToRetry())
        sleep(timePeriodSeconds(period))
        self.assertFalse(ti.timeToRetry())

        self.assertTrue(ti.retry())
        self.assertFalse(ti.timeToRetry())
        sleep(timePeriodSeconds(period))
        self.assertTrue(ti.timeToRetry())

        self.assertTrue(ti.retry())
        self.assertFalse(ti.timeToRetry())
        sleep(timePeriodSeconds(period))
        self.assertFalse(ti.timeToRetry())  # Each retry increases
        sleep(timePeriodSeconds(period))
        self.assertTrue(ti.timeToRetry())

        self.assertFalse(ti.retry())  # Exceeds maximum time
コード例 #44
0
ファイル: convention.py プロジェクト: kquick/Thespian
    def got_convention_register(self, regmsg):
        # Called when remote convention member has sent a
        # ConventionRegister message.  This is first called the leader
        # when the member registers with the leader, and then on the
        # member when the leader responds with same.  Thus the current
        # node could be a member, a potential leader, the current
        # leader, or a potential leader with higher potential than the
        # current leader and which should become the new leader.
        self._sCBStats.inc('Admin Handle Convention Registration')
        if self._invited and not self.conventionLeaderAddr:
            # Lost connection to an invitation-only convention.
            # Cannot join again until another invitation is received.
            return []
        # Remote member may re-register if changing capabilities
        rmsgs = []
        registrant = regmsg.adminAddress
        prereg = getattr(regmsg, 'preRegister',
                         False)  # getattr used; see definition
        existing = self._conventionMembers.find(registrant)
        thesplog('Got Convention %sregistration from %s (%s) (new? %s)',
                 'pre-' if prereg else '',
                 registrant,
                 'first time' if regmsg.firstTime else 're-registering',
                 not existing,
                 level=logging.DEBUG)
        if registrant == self.myAddress:
            # Either remote failed getting an external address and is
            # using 127.0.0.1 or else this is a malicious attempt to
            # make us talk to ourselves.  Ignore it.
            thesplog(
                'Convention registration from %s is an invalid address; ignoring.',
                registrant,
                level=logging.WARNING)
            return rmsgs

        existingPreReg = (
            # existing.preRegOnly
            # or existing.preRegistered
            existing.permanentEntry) if existing else False
        notify = (not existing or existing.preRegOnly) and not prereg

        if regmsg.firstTime or not existing:
            if existing:
                existing = None
                notify = not prereg
                rmsgs.extend(self._remote_system_cleanup(registrant))
            newmember = ConventionMemberData(registrant, regmsg.capabilities,
                                             prereg)
            if prereg or existingPreReg:
                newmember.preRegistered = PreRegistration()
            self._conventionMembers.add(registrant, newmember)
        else:
            existing.refresh(regmsg.capabilities, prereg or existingPreReg)
            if not prereg:
                existing.preRegOnly = False

        if not self.isConventionLeader():
            self._conventionRegistration = ExpirationTimer(
                CONVENTION_REREGISTRATION_PERIOD)
            rmsgs.append(LogAggregator(self.conventionLeaderAddr))

        # Convention Members normally periodically initiate a
        # membership message, to which the leader confirms by
        # responding.
        #if self.isConventionLeader() or prereg or regmsg.firstTime:
        if prereg:
            # If this was a pre-registration, that identifies this
            # system as the "leader" for that remote.  Also, if the
            # remote sent this because it was a pre-registration
            # leader, it doesn't yet have all the member information
            # so the member should respond.
            rmsgs.append(HysteresisCancel(registrant))
            rmsgs.append(TransmitIntent(registrant, ConventionInvite()))
        elif (self.isConventionLeader() or prereg or regmsg.firstTime or \
           (existing and existing.permanentEntry)):
            # If we are the Convention Leader, this would be the point to
            # inform all other registrants of the new registrant.  At
            # present, there is no reciprocity here, so just update the
            # new registrant with the leader's info.
            rmsgs.append(
                TransmitIntent(
                    registrant,
                    ConventionRegister(self.myAddress, self.capabilities)))

        if notify:
            rmsgs.extend(
                self._notifications_of(
                    ActorSystemConventionUpdate(registrant,
                                                regmsg.capabilities, True)))
        return rmsgs
コード例 #45
0
 def testTransmitIntentRetry(self):
     ti = TransmitIntent('addr', 'msg')
     for x in range(MAX_TRANSMIT_RETRIES+1):
         self.assertTrue(ti.retry())
     self.assertFalse(ti.retry())
コード例 #46
0
ファイル: convention.py プロジェクト: kquick/Thespian
 def _notifications_of(self, msg):
     return [
         TransmitIntent(H, msg)
         for H in self._conventionNotificationHandlers
     ]
コード例 #47
0
ファイル: convention.py プロジェクト: kquick/Thespian
    def _remote_system_cleanup(self, registrant):
        """Called when a RemoteActorSystem has exited and all associated
           Actors should be marked as exited and the ActorSystem
           removed from Convention membership.  This is also called on
           a First Time connection from the remote to discard any
           previous connection information.

        """
        thesplog('Convention cleanup or deregistration for %s (known? %s)',
                 registrant,
                 bool(self._conventionMembers.find(registrant)),
                 level=logging.INFO)
        rmsgs = [LostRemote(registrant)]
        cmr = self._conventionMembers.find(registrant)
        if not cmr or cmr.preRegOnly:
            return []

        # Send exited notification to conventionNotificationHandler (if any)
        for each in self._conventionNotificationHandlers:
            rmsgs.append(
                TransmitIntent(
                    each,
                    ActorSystemConventionUpdate(cmr.remoteAddress,
                                                cmr.remoteCapabilities,
                                                False)))  # errors ignored

        # If the remote ActorSystem shutdown gracefully (i.e. sent
        # a Convention Deregistration) then it should not be
        # necessary to shutdown remote Actors (or notify of their
        # shutdown) because the remote ActorSystem should already
        # have caused this to occur.  However, it won't hurt, and
        # it's necessary if the remote ActorSystem did not exit
        # gracefully.

        for lpa, raa in cmr.hasRemoteActors:
            # ignore errors:
            rmsgs.append(TransmitIntent(lpa, ChildActorExited(raa)))
            # n.b. at present, this means that the parent might
            # get duplicate notifications of ChildActorExited; it
            # is expected that Actors can handle this.

        # Remove remote system from conventionMembers
        if not cmr.preRegistered:
            cla = self.conventionLeaderAddr
            self._conventionMembers.rmv(registrant)
            if registrant == cla:
                if self._invited:
                    # Don't clear invited: once invited, that
                    # perpetually indicates this should be only a
                    # member and never a leader.
                    self._conventionAddress = [None]
                else:
                    rmsgs.extend(self.setup_convention())
        else:
            # This conventionMember needs to stay because the
            # current system needs to continue issuing
            # registration pings.  By setting the registryValid
            # expiration to forever, this member won't re-time-out
            # and will therefore be otherwise ignored... until it
            # registers again at which point the membership will
            # be updated with new settings.
            cmr.registryValid = ExpirationTimer(None)
            cmr.preRegOnly = True

        return rmsgs + [HysteresisCancel(registrant)]
コード例 #48
0
 def test_sendIntentToTransport(self):
     testTrans = FakeTransport()
     testIntent = TransmitIntent(ActorAddress(None), 'message',
                                 self.successCB, self.failureCB)
     testTrans.scheduleTransmit(None, testIntent)
     assert 1 == len(testTrans.intents)
コード例 #49
0
    def testTwentySendsSameAddressSameMessageTypeSendAfterDelay(self):
        self.sends = []
        min_h_period = timedelta(milliseconds=2)
        max_h_period = timedelta(milliseconds=20)
        h_rate = 2
        hs = HysteresisDelaySender(self.send,
                                   hysteresis_min_period=min_h_period,
                                   hysteresis_max_period=max_h_period,
                                   hysteresis_rate=h_rate)

        # Create the messages to send
        intents = [TransmitIntent('addr1', 'msg1')]
        for num in range(20):
            intents.append(TransmitIntent('addr1', 'msg'))

        # Send all intents in rapid succession
        for each in intents:
            hs.sendWithHysteresis(each)

        # First was sent immediately, all others are delayed
        assert 1 == len(getattr(self, 'sends', []))
        assert intents[0] == self.sends[0]

        # The hysteresis delay should be maxed out
        t1 = hs.delay.remaining()
        assert timedelta(seconds=0) != t1
        assert max_h_period >= t1

        # Wait the delay period and then check, which should send the
        # (latest) queued messages.  Because all the messages and
        # target addresses are identical, this will actually only send
        # a single message.
        sleep(hs.delay.remainingSeconds())
        hs.checkSends()
        assert 2 == len(getattr(self, 'sends', []))

        # Verify that although the queued messages were sent, the
        # hysteresis delay is not yet back to zero and additional
        # sends are still blocked.
        assert not hs.delay.expired()  # refreshed and reduced in checkSends
        hs.sendWithHysteresis(intents[0])
        t1 = hs.delay.remaining()  # send attempt probably bumped this up again
        assert 2 == len(getattr(self, 'sends', []))
        assert intents[0] == self.sends[0]
        assert intents[-1] == self.sends[1]

        # Verify that the hysteresis delay keeps dropping and
        # eventually gets back to zero.  After a drop, any pending
        # sends that were blocked should be sent.
        nsent = 2  # after first wait, checkSends will send the one just queued...
        for x in range(100):  # don't loop forever
            if hs.delay.expired(): break
            t2 = hs.delay.remaining()
            assert timedelta(seconds=0) != t2
            assert (x, t2) < (x, t1)
            assert nsent == len(getattr(self, 'sends', []))
            sleep(hs.delay.remainingSeconds())
            t1 = t2
            hs.checkSends()
            if nsent == 2:
                # All queued sends should now have been sent, but
                # since they are all the same, there was only one more
                # actual send.
                nsent = 3

        # Now verify hysteresis sender is back to the original state
        assert 3 == len(getattr(self, 'sends', []))

        hs.sendWithHysteresis(intents[1])

        assert 4 == len(getattr(self, 'sends', []))
        assert intents[0] == self.sends[0]
        assert intents[-1] == self.sends[1]
        assert intents[0] == self.sends[2]
        assert intents[1] == self.sends[3]

        # Verify that all intents got completed even though some were
        # duplicates and not actually sent.
        for each in intents:
            assert each.result == SendStatus.Sent